A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[CI/Build] [1/3] Reorganize entrypoints tests (#5526)
Cyrus Leung committed
Jun 27, 2024 at 20:43 UTC
e9d32d077da2f914c965a66dd2fdfc77c50d117f
3 files changed
+896
-774
tests/entrypoints/test_openai_chat.py
new
+875
@@ -0,0 +1,875 @@
1
+# imports for guided decoding tests
2
+import json
3
+import re
4
+from typing import List
5
+
6
+import jsonschema
7
+import openai # use the official client for correctness check
8
+import pytest
9
+# using Ray for overall ease of process management, parallel requests,
10
+# and debugging.
11
+import ray
12
+import torch
13
+# downloading lora to test lora requests
14
+from huggingface_hub import snapshot_download
15
+from openai import BadRequestError
16
+
17
+from ..utils import VLLM_PATH, RemoteOpenAIServer
18
+
19
+# any model with a chat template should work here
20
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
21
+# technically this needs Mistral-7B-v0.1 as base, but we're not testing
22
+# generation quality here
23
+LORA_NAME = "typeof/zephyr-7b-beta-lora"
24
+
25
+TEST_SCHEMA = {
26
+ "type": "object",
27
+ "properties": {
28
+ "name": {
29
+ "type": "string"
30
+ },
31
+ "age": {
32
+ "type": "integer"
33
+ },
34
+ "skills": {
35
+ "type": "array",
36
+ "items": {
37
+ "type": "string",
38
+ "maxLength": 10
39
+ },
40
+ "minItems": 3
41
+ },
42
+ "work history": {
43
+ "type": "array",
44
+ "items": {
45
+ "type": "object",
46
+ "properties": {
47
+ "company": {
48
+ "type": "string"
49
+ },
50
+ "duration": {
51
+ "type": "string"
52
+ },
53
+ "position": {
54
+ "type": "string"
55
+ }
56
+ },
57
+ "required": ["company", "position"]
58
+ }
59
+ }
60
+ },
61
+ "required": ["name", "age", "skills", "work history"]
62
+}
63
+
64
+TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
65
+ r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
66
+
67
+TEST_CHOICE = [
68
+ "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
69
+ "Swift", "Kotlin"
70
+]
71
+
72
+pytestmark = pytest.mark.openai
73
+
74
+
75
+@pytest.fixture(scope="session")
76
+def zephyr_lora_files():
77
+ return snapshot_download(repo_id=LORA_NAME)
78
+
79
+
80
+@pytest.fixture(scope="module")
81
+def ray_ctx():
82
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
83
+ yield
84
+ ray.shutdown()
85
+
86
+
87
+@pytest.fixture(scope="module")
88
+def server(zephyr_lora_files, ray_ctx):
89
+ return RemoteOpenAIServer([
90
+ "--model",
91
+ MODEL_NAME,
92
+ # use half precision for speed and memory savings in CI environment
93
+ "--dtype",
94
+ "bfloat16",
95
+ "--max-model-len",
96
+ "8192",
97
+ "--enforce-eager",
98
+ # lora config below
99
+ "--enable-lora",
100
+ "--lora-modules",
101
+ f"zephyr-lora={zephyr_lora_files}",
102
+ f"zephyr-lora2={zephyr_lora_files}",
103
+ "--max-lora-rank",
104
+ "64",
105
+ "--max-cpu-loras",
106
+ "2",
107
+ "--max-num-seqs",
108
+ "128",
109
+ ])
110
+
111
+
112
+@pytest.fixture(scope="module")
113
+def client(server):
114
+ return server.get_async_client()
115
+
116
+
117
+@pytest.mark.asyncio
118
+@pytest.mark.parametrize(
119
+ # first test base model, then test loras
120
+ "model_name",
121
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
122
+)
123
+async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
124
+ messages = [{
125
+ "role": "system",
126
+ "content": "you are a helpful assistant"
127
+ }, {
128
+ "role": "user",
129
+ "content": "what is 1+1?"
130
+ }]
131
+
132
+ chat_completion = await client.chat.completions.create(model=model_name,
133
+ messages=messages,
134
+ max_tokens=5,
135
+ temperature=0.0,
136
+ logprobs=False)
137
+
138
+ choice = chat_completion.choices[0]
139
+ assert choice.logprobs is None
140
+
141
+
142
+@pytest.mark.asyncio
143
+@pytest.mark.parametrize(
144
+ # just test 1 lora hereafter
145
+ "model_name",
146
+ [MODEL_NAME, "zephyr-lora"],
147
+)
148
+async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
149
+ messages = [{
150
+ "role": "system",
151
+ "content": "you are a helpful assistant"
152
+ }, {
153
+ "role": "user",
154
+ "content": "what is 1+1?"
155
+ }]
156
+
157
+ chat_completion = await client.chat.completions.create(model=model_name,
158
+ messages=messages,
159
+ max_tokens=5,
160
+ temperature=0.0,
161
+ logprobs=True,
162
+ top_logprobs=0)
163
+
164
+ choice = chat_completion.choices[0]
165
+ assert choice.logprobs is not None
166
+ assert choice.logprobs.content is not None
167
+ assert len(choice.logprobs.content[0].top_logprobs) == 0
168
+
169
+
170
+@pytest.mark.asyncio
171
+@pytest.mark.parametrize(
172
+ "model_name",
173
+ [MODEL_NAME, "zephyr-lora"],
174
+)
175
+async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
176
+ messages = [{
177
+ "role": "system",
178
+ "content": "you are a helpful assistant"
179
+ }, {
180
+ "role": "user",
181
+ "content": "what is 1+1?"
182
+ }]
183
+
184
+ chat_completion = await client.chat.completions.create(model=model_name,
185
+ messages=messages,
186
+ max_tokens=5,
187
+ temperature=0.0,
188
+ logprobs=True,
189
+ top_logprobs=5)
190
+
191
+ choice = chat_completion.choices[0]
192
+ assert choice.logprobs is not None
193
+ assert choice.logprobs.content is not None
194
+ assert len(choice.logprobs.content[0].top_logprobs) == 5
195
+
196
+
197
+@pytest.mark.asyncio
198
+@pytest.mark.parametrize(
199
+ "model_name",
200
+ [MODEL_NAME, "zephyr-lora"],
201
+)
202
+async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
203
+ model_name: str):
204
+ messages = [{
205
+ "role": "system",
206
+ "content": "you are a helpful assistant"
207
+ }, {
208
+ "role": "user",
209
+ "content": "what is 1+1?"
210
+ }]
211
+
212
+ # Default max_logprobs is 20, so this should raise an error
213
+ with pytest.raises((openai.BadRequestError, openai.APIError)):
214
+ stream = await client.chat.completions.create(model=model_name,
215
+ messages=messages,
216
+ max_tokens=10,
217
+ logprobs=True,
218
+ top_logprobs=21,
219
+ stream=True)
220
+ async for chunk in stream:
221
+ ...
222
+
223
+ with pytest.raises(openai.BadRequestError):
224
+ await client.chat.completions.create(model=model_name,
225
+ messages=messages,
226
+ max_tokens=10,
227
+ logprobs=True,
228
+ top_logprobs=30,
229
+ stream=False)
230
+
231
+ # the server should still work afterwards
232
+ chat_completion = await client.chat.completions.create(model=model_name,
233
+ messages=messages,
234
+ max_tokens=10,
235
+ stream=False)
236
+ message = chat_completion.choices[0].message
237
+ assert message.content is not None and len(message.content) >= 0
238
+
239
+
240
+@pytest.mark.asyncio
241
+@pytest.mark.parametrize(
242
+ "model_name",
243
+ [MODEL_NAME, "zephyr-lora"],
244
+)
245
+async def test_single_chat_session(client: openai.AsyncOpenAI,
246
+ model_name: str):
247
+ messages = [{
248
+ "role": "system",
249
+ "content": "you are a helpful assistant"
250
+ }, {
251
+ "role": "user",
252
+ "content": "what is 1+1?"
253
+ }]
254
+
255
+ # test single completion
256
+ chat_completion = await client.chat.completions.create(model=model_name,
257
+ messages=messages,
258
+ max_tokens=10,
259
+ logprobs=True,
260
+ top_logprobs=5)
261
+ assert chat_completion.id is not None
262
+ assert len(chat_completion.choices) == 1
263
+
264
+ choice = chat_completion.choices[0]
265
+ assert choice.finish_reason == "length"
266
+ assert chat_completion.usage == openai.types.CompletionUsage(
267
+ completion_tokens=10, prompt_tokens=37, total_tokens=47)
268
+
269
+ message = choice.message
270
+ assert message.content is not None and len(message.content) >= 10
271
+ assert message.role == "assistant"
272
+ messages.append({"role": "assistant", "content": message.content})
273
+
274
+ # test multi-turn dialogue
275
+ messages.append({"role": "user", "content": "express your result in json"})
276
+ chat_completion = await client.chat.completions.create(
277
+ model=model_name,
278
+ messages=messages,
279
+ max_tokens=10,
280
+ )
281
+ message = chat_completion.choices[0].message
282
+ assert message.content is not None and len(message.content) >= 0
283
+
284
+
285
+@pytest.mark.asyncio
286
+@pytest.mark.parametrize(
287
+ # just test 1 lora hereafter
288
+ "model_name",
289
+ [MODEL_NAME, "zephyr-lora"],
290
+)
291
+async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
292
+ messages = [{
293
+ "role": "system",
294
+ "content": "you are a helpful assistant"
295
+ }, {
296
+ "role": "user",
297
+ "content": "what is 1+1?"
298
+ }]
299
+
300
+ # test single completion
301
+ chat_completion = await client.chat.completions.create(
302
+ model=model_name,
303
+ messages=messages,
304
+ max_tokens=10,
305
+ temperature=0.0,
306
+ )
307
+ output = chat_completion.choices[0].message.content
308
+ stop_reason = chat_completion.choices[0].finish_reason
309
+
310
+ # test streaming
311
+ stream = await client.chat.completions.create(
312
+ model=model_name,
313
+ messages=messages,
314
+ max_tokens=10,
315
+ temperature=0.0,
316
+ stream=True,
317
+ )
318
+ chunks: List[str] = []
319
+ finish_reason_count = 0
320
+ async for chunk in stream:
321
+ delta = chunk.choices[0].delta
322
+ if delta.role:
323
+ assert delta.role == "assistant"
324
+ if delta.content:
325
+ chunks.append(delta.content)
326
+ if chunk.choices[0].finish_reason is not None:
327
+ finish_reason_count += 1
328
+ # finish reason should only return in last block
329
+ assert finish_reason_count == 1
330
+ assert chunk.choices[0].finish_reason == stop_reason
331
+ assert delta.content
332
+ assert "".join(chunks) == output
333
+
334
+
335
+@pytest.mark.asyncio
336
+@pytest.mark.parametrize(
337
+ "model_name",
338
+ ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
339
+)
340
+async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
341
+ model_name: str):
342
+ messages = [{
343
+ "role": "system",
344
+ "content": "You are a helpful assistant."
345
+ }, {
346
+ "role": "user",
347
+ "content": "What is the capital of France?"
348
+ }]
349
+
350
+ # Test stream=True, stream_options={"include_usage": False}
351
+ stream = await client.chat.completions.create(
352
+ model=model_name,
353
+ messages=messages,
354
+ max_tokens=10,
355
+ temperature=0.0,
356
+ stream=True,
357
+ stream_options={"include_usage": False})
358
+ async for chunk in stream:
359
+ assert chunk.usage is None
360
+
361
+ # Test stream=True, stream_options={"include_usage": True}
362
+ stream = await client.chat.completions.create(
363
+ model=model_name,
364
+ messages=messages,
365
+ max_tokens=10,
366
+ temperature=0.0,
367
+ stream=True,
368
+ stream_options={"include_usage": True})
369
+
370
+ async for chunk in stream:
371
+ if chunk.choices[0].finish_reason is None:
372
+ assert chunk.usage is None
373
+ else:
374
+ assert chunk.usage is None
375
+ final_chunk = await stream.__anext__()
376
+ assert final_chunk.usage is not None
377
+ assert final_chunk.usage.prompt_tokens > 0
378
+ assert final_chunk.usage.completion_tokens > 0
379
+ assert final_chunk.usage.total_tokens == (
380
+ final_chunk.usage.prompt_tokens +
381
+ final_chunk.usage.completion_tokens)
382
+ assert final_chunk.choices == []
383
+
384
+ # Test stream=False, stream_options={"include_usage": None}
385
+ with pytest.raises(BadRequestError):
386
+ await client.chat.completions.create(
387
+ model=model_name,
388
+ messages=messages,
389
+ max_tokens=10,
390
+ temperature=0.0,
391
+ stream=False,
392
+ stream_options={"include_usage": None})
393
+
394
+ # Test stream=False, stream_options={"include_usage": True}
395
+ with pytest.raises(BadRequestError):
396
+ await client.chat.completions.create(
397
+ model=model_name,
398
+ messages=messages,
399
+ max_tokens=10,
400
+ temperature=0.0,
401
+ stream=False,
402
+ stream_options={"include_usage": True})
403
+
404
+
405
+# NOTE: Not sure why, but when I place this after `test_guided_regex_chat`
406
+# (i.e. using the same ordering as in the Completions API tests), the test
407
+# will fail on the second `guided_decoding_backend` even when I swap their order
408
+# (ref: https://github.com/vllm-project/vllm/pull/5526#issuecomment-2173772256)
409
+@pytest.mark.asyncio
410
+@pytest.mark.parametrize("guided_decoding_backend",
411
+ ["outlines", "lm-format-enforcer"])
412
+async def test_guided_choice_chat(client: openai.AsyncOpenAI,
413
+ guided_decoding_backend: str):
414
+ messages = [{
415
+ "role": "system",
416
+ "content": "you are a helpful assistant"
417
+ }, {
418
+ "role":
419
+ "user",
420
+ "content":
421
+ "The best language for type-safe systems programming is "
422
+ }]
423
+ chat_completion = await client.chat.completions.create(
424
+ model=MODEL_NAME,
425
+ messages=messages,
426
+ max_tokens=10,
427
+ extra_body=dict(guided_choice=TEST_CHOICE,
428
+ guided_decoding_backend=guided_decoding_backend))
429
+ choice1 = chat_completion.choices[0].message.content
430
+ assert choice1 in TEST_CHOICE
431
+
432
+ messages.append({"role": "assistant", "content": choice1})
433
+ messages.append({
434
+ "role": "user",
435
+ "content": "I disagree, pick another one"
436
+ })
437
+ chat_completion = await client.chat.completions.create(
438
+ model=MODEL_NAME,
439
+ messages=messages,
440
+ max_tokens=10,
441
+ extra_body=dict(guided_choice=TEST_CHOICE,
442
+ guided_decoding_backend=guided_decoding_backend))
443
+ choice2 = chat_completion.choices[0].message.content
444
+ assert choice2 in TEST_CHOICE
445
+ assert choice1 != choice2
446
+
447
+
448
+@pytest.mark.asyncio
449
+@pytest.mark.parametrize("guided_decoding_backend",
450
+ ["outlines", "lm-format-enforcer"])
451
+async def test_guided_json_chat(client: openai.AsyncOpenAI,
452
+ guided_decoding_backend: str):
453
+ messages = [{
454
+ "role": "system",
455
+ "content": "you are a helpful assistant"
456
+ }, {
457
+ "role":
458
+ "user",
459
+ "content":
460
+ f"Give an example JSON for an employee profile that "
461
+ f"fits this schema: {TEST_SCHEMA}"
462
+ }]
463
+ chat_completion = await client.chat.completions.create(
464
+ model=MODEL_NAME,
465
+ messages=messages,
466
+ max_tokens=1000,
467
+ extra_body=dict(guided_json=TEST_SCHEMA,
468
+ guided_decoding_backend=guided_decoding_backend))
469
+ message = chat_completion.choices[0].message
470
+ assert message.content is not None
471
+ json1 = json.loads(message.content)
472
+ jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
473
+
474
+ messages.append({"role": "assistant", "content": message.content})
475
+ messages.append({
476
+ "role":
477
+ "user",
478
+ "content":
479
+ "Give me another one with a different name and age"
480
+ })
481
+ chat_completion = await client.chat.completions.create(
482
+ model=MODEL_NAME,
483
+ messages=messages,
484
+ max_tokens=1000,
485
+ extra_body=dict(guided_json=TEST_SCHEMA,
486
+ guided_decoding_backend=guided_decoding_backend))
487
+ message = chat_completion.choices[0].message
488
+ assert message.content is not None
489
+ json2 = json.loads(message.content)
490
+ jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
491
+ assert json1["name"] != json2["name"]
492
+ assert json1["age"] != json2["age"]
493
+
494
+
495
+@pytest.mark.asyncio
496
+@pytest.mark.parametrize("guided_decoding_backend",
497
+ ["outlines", "lm-format-enforcer"])
498
+async def test_guided_regex_chat(client: openai.AsyncOpenAI,
499
+ guided_decoding_backend: str):
500
+ messages = [{
501
+ "role": "system",
502
+ "content": "you are a helpful assistant"
503
+ }, {
504
+ "role":
505
+ "user",
506
+ "content":
507
+ f"Give an example IP address with this regex: {TEST_REGEX}"
508
+ }]
509
+ chat_completion = await client.chat.completions.create(
510
+ model=MODEL_NAME,
511
+ messages=messages,
512
+ max_tokens=20,
513
+ extra_body=dict(guided_regex=TEST_REGEX,
514
+ guided_decoding_backend=guided_decoding_backend))
515
+ ip1 = chat_completion.choices[0].message.content
516
+ assert ip1 is not None
517
+ assert re.fullmatch(TEST_REGEX, ip1) is not None
518
+
519
+ messages.append({"role": "assistant", "content": ip1})
520
+ messages.append({"role": "user", "content": "Give me a different one"})
521
+ chat_completion = await client.chat.completions.create(
522
+ model=MODEL_NAME,
523
+ messages=messages,
524
+ max_tokens=20,
525
+ extra_body=dict(guided_regex=TEST_REGEX,
526
+ guided_decoding_backend=guided_decoding_backend))
527
+ ip2 = chat_completion.choices[0].message.content
528
+ assert ip2 is not None
529
+ assert re.fullmatch(TEST_REGEX, ip2) is not None
530
+ assert ip1 != ip2
531
+
532
+
533
+@pytest.mark.asyncio
534
+async def test_guided_decoding_type_error(client: openai.AsyncOpenAI):
535
+ messages = [{
536
+ "role": "system",
537
+ "content": "you are a helpful assistant"
538
+ }, {
539
+ "role":
540
+ "user",
541
+ "content":
542
+ "The best language for type-safe systems programming is "
543
+ }]
544
+
545
+ with pytest.raises(openai.BadRequestError):
546
+ _ = await client.chat.completions.create(model=MODEL_NAME,
547
+ messages=messages,
548
+ extra_body=dict(guided_regex={
549
+ 1: "Python",
550
+ 2: "C++"
551
+ }))
552
+
553
+
554
+@pytest.mark.asyncio
555
+@pytest.mark.parametrize("guided_decoding_backend",
556
+ ["outlines", "lm-format-enforcer"])
557
+async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
558
+ guided_decoding_backend: str):
559
+ messages = [{
560
+ "role": "system",
561
+ "content": "you are a helpful assistant"
562
+ }, {
563
+ "role":
564
+ "user",
565
+ "content":
566
+ "The best language for type-safe systems programming is "
567
+ }]
568
+ chat_completion = await client.chat.completions.create(
569
+ model=MODEL_NAME,
570
+ messages=messages,
571
+ max_tokens=10,
572
+ logprobs=True,
573
+ top_logprobs=5,
574
+ extra_body=dict(guided_choice=TEST_CHOICE,
575
+ guided_decoding_backend=guided_decoding_backend))
576
+
577
+ assert chat_completion.choices[0].logprobs is not None
578
+ assert chat_completion.choices[0].logprobs.content is not None
579
+ top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
580
+
581
+ # -9999.0 is the minimum logprob returned by OpenAI
582
+ for item in top_logprobs:
583
+ assert item.logprob >= -9999.0, f"Failed (top_logprobs={top_logprobs})"
584
+
585
+
586
+@pytest.mark.asyncio
587
+@pytest.mark.parametrize("guided_decoding_backend",
588
+ ["outlines", "lm-format-enforcer"])
589
+async def test_named_tool_use(client: openai.AsyncOpenAI,
590
+ guided_decoding_backend: str):
591
+ messages = [{
592
+ "role": "system",
593
+ "content": "you are a helpful assistant"
594
+ }, {
595
+ "role":
596
+ "user",
597
+ "content":
598
+ f"Give an example JSON for an employee profile that "
599
+ f"fits this schema: {TEST_SCHEMA}"
600
+ }]
601
+
602
+ # non-streaming
603
+
604
+ chat_completion = await client.chat.completions.create(
605
+ model=MODEL_NAME,
606
+ messages=messages,
607
+ max_tokens=1000,
608
+ tools=[{
609
+ "type": "function",
610
+ "function": {
611
+ "name": "dummy_function_name",
612
+ "description": "This is a dummy function",
613
+ "parameters": TEST_SCHEMA
614
+ }
615
+ }],
616
+ tool_choice={
617
+ "type": "function",
618
+ "function": {
619
+ "name": "dummy_function_name"
620
+ }
621
+ })
622
+ message = chat_completion.choices[0].message
623
+ assert len(message.content) == 0
624
+ json_string = message.tool_calls[0].function.arguments
625
+ json1 = json.loads(json_string)
626
+ jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
627
+
628
+ messages.append({"role": "assistant", "content": json_string})
629
+ messages.append({
630
+ "role":
631
+ "user",
632
+ "content":
633
+ "Give me another one with a different name and age"
634
+ })
635
+
636
+ # streaming
637
+
638
+ stream = await client.chat.completions.create(
639
+ model=MODEL_NAME,
640
+ messages=messages,
641
+ max_tokens=1000,
642
+ tools=[{
643
+ "type": "function",
644
+ "function": {
645
+ "name": "dummy_function_name",
646
+ "description": "This is a dummy function",
647
+ "parameters": TEST_SCHEMA
648
+ }
649
+ }],
650
+ tool_choice={
651
+ "type": "function",
652
+ "function": {
653
+ "name": "dummy_function_name"
654
+ }
655
+ },
656
+ stream=True)
657
+
658
+ output = []
659
+ finish_reason_count = 0
660
+ async for chunk in stream:
661
+ delta = chunk.choices[0].delta
662
+ if delta.role:
663
+ assert delta.role == "assistant"
664
+ assert delta.content is None or len(delta.content) == 0
665
+ if delta.tool_calls:
666
+ output.append(delta.tool_calls[0].function.arguments)
667
+ if chunk.choices[0].finish_reason is not None:
668
+ finish_reason_count += 1
669
+ # finish reason should only return in last block
670
+ assert finish_reason_count == 1
671
+ json2 = json.loads("".join(output))
672
+ jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
673
+ assert json1["name"] != json2["name"]
674
+ assert json1["age"] != json2["age"]
675
+
676
+
677
+@pytest.mark.asyncio
678
+@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
679
+async def test_required_tool_use_not_yet_supported(
680
+ client: openai.AsyncOpenAI, guided_decoding_backend: str):
681
+ messages = [{
682
+ "role": "system",
683
+ "content": "you are a helpful assistant"
684
+ }, {
685
+ "role":
686
+ "user",
687
+ "content":
688
+ f"Give an example JSON for an employee profile that "
689
+ f"fits this schema: {TEST_SCHEMA}"
690
+ }]
691
+
692
+ with pytest.raises(openai.BadRequestError):
693
+ await client.chat.completions.create(
694
+ model=MODEL_NAME,
695
+ messages=messages,
696
+ max_tokens=1000,
697
+ tools=[{
698
+ "type": "function",
699
+ "function": {
700
+ "name": "dummy_function_name",
701
+ "description": "This is a dummy function",
702
+ "parameters": TEST_SCHEMA
703
+ }
704
+ }],
705
+ tool_choice="required")
706
+
707
+ with pytest.raises(openai.BadRequestError):
708
+ await client.chat.completions.create(
709
+ model=MODEL_NAME,
710
+ messages=messages,
711
+ max_tokens=1000,
712
+ tools=[{
713
+ "type": "function",
714
+ "function": {
715
+ "name": "dummy_function_name",
716
+ "description": "This is a dummy function",
717
+ "parameters": TEST_SCHEMA
718
+ }
719
+ }],
720
+ tool_choice="auto")
721
+
722
+
723
+@pytest.mark.asyncio
724
+@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
725
+async def test_inconsistent_tool_choice_and_tools(
726
+ client: openai.AsyncOpenAI, guided_decoding_backend: str):
727
+ messages = [{
728
+ "role": "system",
729
+ "content": "you are a helpful assistant"
730
+ }, {
731
+ "role":
732
+ "user",
733
+ "content":
734
+ f"Give an example JSON for an employee profile that "
735
+ f"fits this schema: {TEST_SCHEMA}"
736
+ }]
737
+
738
+ with pytest.raises(openai.BadRequestError):
739
+ await client.chat.completions.create(model=MODEL_NAME,
740
+ messages=messages,
741
+ max_tokens=1000,
742
+ tool_choice={
743
+ "type": "function",
744
+ "function": {
745
+ "name":
746
+ "dummy_function_name"
747
+ }
748
+ })
749
+
750
+ with pytest.raises(openai.BadRequestError):
751
+ await client.chat.completions.create(
752
+ model=MODEL_NAME,
753
+ messages=messages,
754
+ max_tokens=1000,
755
+ tools=[{
756
+ "type": "function",
757
+ "function": {
758
+ "name": "dummy_function_name",
759
+ "description": "This is a dummy function",
760
+ "parameters": TEST_SCHEMA
761
+ }
762
+ }],
763
+ tool_choice={
764
+ "type": "function",
765
+ "function": {
766
+ "name": "nondefined_function_name"
767
+ }
768
+ })
769
+
770
+
771
+@pytest.mark.asyncio
772
+async def test_response_format_json_object(client: openai.AsyncOpenAI):
773
+ for _ in range(2):
774
+ resp = await client.chat.completions.create(
775
+ model=MODEL_NAME,
776
+ messages=[{
777
+ "role":
778
+ "user",
779
+ "content": ('what is 1+1? please respond with a JSON object, '
780
+ 'the format is {"result": 2}')
781
+ }],
782
+ response_format={"type": "json_object"})
783
+
784
+ content = resp.choices[0].message.content
785
+ assert content is not None
786
+
787
+ loaded = json.loads(content)
788
+ assert loaded == {"result": 2}, loaded
789
+
790
+
791
+@pytest.mark.asyncio
792
+async def test_extra_fields(client: openai.AsyncOpenAI):
793
+ with pytest.raises(BadRequestError) as exc_info:
794
+ await client.chat.completions.create(
795
+ model=MODEL_NAME,
796
+ messages=[{
797
+ "role": "system",
798
+ "content": "You are a helpful assistant.",
799
+ "extra_field": "0",
800
+ }], # type: ignore
801
+ temperature=0,
802
+ seed=0)
803
+
804
+ assert "extra_forbidden" in exc_info.value.message
805
+
806
+
807
+@pytest.mark.asyncio
808
+async def test_complex_message_content(client: openai.AsyncOpenAI):
809
+ resp = await client.chat.completions.create(
810
+ model=MODEL_NAME,
811
+ messages=[{
812
+ "role":
813
+ "user",
814
+ "content": [{
815
+ "type":
816
+ "text",
817
+ "text":
818
+ "what is 1+1? please provide the result without any other text."
819
+ }]
820
+ }],
821
+ temperature=0,
822
+ seed=0)
823
+ content = resp.choices[0].message.content
824
+ assert content == "2"
825
+
826
+
827
+@pytest.mark.asyncio
828
+async def test_custom_role(client: openai.AsyncOpenAI):
829
+ # Not sure how the model handles custom roles so we just check that
830
+ # both string and complex message content are handled in the same way
831
+
832
+ resp1 = await client.chat.completions.create(
833
+ model=MODEL_NAME,
834
+ messages=[{
835
+ "role": "my-custom-role",
836
+ "content": "what is 1+1?",
837
+ }], # type: ignore
838
+ temperature=0,
839
+ seed=0)
840
+
841
+ resp2 = await client.chat.completions.create(
842
+ model=MODEL_NAME,
843
+ messages=[{
844
+ "role": "my-custom-role",
845
+ "content": [{
846
+ "type": "text",
847
+ "text": "what is 1+1?"
848
+ }]
849
+ }], # type: ignore
850
+ temperature=0,
851
+ seed=0)
852
+
853
+ content1 = resp1.choices[0].message.content
854
+ content2 = resp2.choices[0].message.content
855
+ assert content1 == content2
856
+
857
+
858
+@pytest.mark.asyncio
859
+async def test_long_seed(client: openai.AsyncOpenAI):
860
+ for seed in [
861
+ torch.iinfo(torch.long).min - 1,
862
+ torch.iinfo(torch.long).max + 1
863
+ ]:
864
+ with pytest.raises(BadRequestError) as exc_info:
865
+ await client.chat.completions.create(
866
+ model=MODEL_NAME,
867
+ messages=[{
868
+ "role": "system",
869
+ "content": "You are a helpful assistant.",
870
+ }],
871
+ temperature=0,
872
+ seed=seed)
873
+
874
+ assert ("greater_than_equal" in exc_info.value.message
875
+ or "less_than_equal" in exc_info.value.message)
tests/entrypoints/test_openai_server.py
+19
-772
@@ -10,7 +10,6 @@ import pytest
10
# and debugging.
11
import ray
12
import requests
13
-import torch
13
# downloading lora to test lora requests
14
from huggingface_hub import snapshot_download
15
from openai import BadRequestError
@@ -117,6 +116,7 @@ def client(server):
116
return server.get_async_client()
117
118
119
+@pytest.mark.asyncio
120
async def test_check_models(client: openai.AsyncOpenAI):
121
models = await client.models.list()
122
models = models.data
@@ -266,174 +266,6 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
266
assert len(completion.choices[0].text) >= 0
267
268
269
-@pytest.mark.asyncio
270
-@pytest.mark.parametrize(
271
- # first test base model, then test loras
272
- "model_name",
273
- [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
274
-)
275
-async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
276
- messages = [{
277
- "role": "system",
278
- "content": "you are a helpful assistant"
279
- }, {
280
- "role": "user",
281
- "content": "what is 1+1?"
282
- }]
283
-
284
- chat_completion = await client.chat.completions.create(model=model_name,
285
- messages=messages,
286
- max_tokens=5,
287
- temperature=0.0,
288
- logprobs=False)
289
-
290
- choice = chat_completion.choices[0]
291
- assert choice.logprobs is None
292
-
293
-
294
-@pytest.mark.asyncio
295
-@pytest.mark.parametrize(
296
- # just test 1 lora hereafter
297
- "model_name",
298
- [MODEL_NAME, "zephyr-lora"],
299
-)
300
-async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
301
- messages = [{
302
- "role": "system",
303
- "content": "you are a helpful assistant"
304
- }, {
305
- "role": "user",
306
- "content": "what is 1+1?"
307
- }]
308
-
309
- chat_completion = await client.chat.completions.create(model=model_name,
310
- messages=messages,
311
- max_tokens=5,
312
- temperature=0.0,
313
- logprobs=True,
314
- top_logprobs=0)
315
-
316
- choice = chat_completion.choices[0]
317
- assert choice.logprobs is not None
318
- assert choice.logprobs.content is not None
319
- assert len(choice.logprobs.content[0].top_logprobs) == 0
320
-
321
-
322
-@pytest.mark.asyncio
323
-@pytest.mark.parametrize(
324
- "model_name",
325
- [MODEL_NAME, "zephyr-lora"],
326
-)
327
-async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
328
- messages = [{
329
- "role": "system",
330
- "content": "you are a helpful assistant"
331
- }, {
332
- "role": "user",
333
- "content": "what is 1+1?"
334
- }]
335
-
336
- chat_completion = await client.chat.completions.create(model=model_name,
337
- messages=messages,
338
- max_tokens=5,
339
- temperature=0.0,
340
- logprobs=True,
341
- top_logprobs=5)
342
-
343
- choice = chat_completion.choices[0]
344
- assert choice.logprobs is not None
345
- assert choice.logprobs.content is not None
346
- assert len(choice.logprobs.content[0].top_logprobs) == 5
347
-
348
-
349
-@pytest.mark.asyncio
350
-@pytest.mark.parametrize(
351
- "model_name",
352
- [MODEL_NAME, "zephyr-lora"],
353
-)
354
-async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
355
- model_name: str):
356
- messages = [{
357
- "role": "system",
358
- "content": "you are a helpful assistant"
359
- }, {
360
- "role": "user",
361
- "content": "what is 1+1?"
362
- }]
363
-
364
- # Default max_logprobs is 20, so this should raise an error
365
- with pytest.raises((openai.BadRequestError, openai.APIError)):
366
- stream = await client.chat.completions.create(model=model_name,
367
- messages=messages,
368
- max_tokens=10,
369
- logprobs=True,
370
- top_logprobs=21,
371
- stream=True)
372
- async for chunk in stream:
373
- ...
374
-
375
- with pytest.raises(openai.BadRequestError):
376
- await client.chat.completions.create(model=model_name,
377
- messages=messages,
378
- max_tokens=10,
379
- logprobs=True,
380
- top_logprobs=30,
381
- stream=False)
382
-
383
- # the server should still work afterwards
384
- chat_completion = await client.chat.completions.create(model=model_name,
385
- messages=messages,
386
- max_tokens=10,
387
- stream=False)
388
- message = chat_completion.choices[0].message
389
- assert message.content is not None and len(message.content) >= 0
390
-
391
-
392
-@pytest.mark.asyncio
393
-@pytest.mark.parametrize(
394
- "model_name",
395
- [MODEL_NAME, "zephyr-lora"],
396
-)
397
-async def test_single_chat_session(client: openai.AsyncOpenAI,
398
- model_name: str):
399
- messages = [{
400
- "role": "system",
401
- "content": "you are a helpful assistant"
402
- }, {
403
- "role": "user",
404
- "content": "what is 1+1?"
405
- }]
406
-
407
- # test single completion
408
- chat_completion = await client.chat.completions.create(model=model_name,
409
- messages=messages,
410
- max_tokens=10,
411
- logprobs=True,
412
- top_logprobs=5)
413
- assert chat_completion.id is not None
414
- assert len(chat_completion.choices) == 1
415
-
416
- choice = chat_completion.choices[0]
417
- assert choice.finish_reason == "length"
418
- assert chat_completion.usage == openai.types.CompletionUsage(
419
- completion_tokens=10, prompt_tokens=37, total_tokens=47)
420
-
421
- message = choice.message
422
- assert message.content is not None and len(message.content) >= 10
423
- assert message.role == "assistant"
424
- messages.append({"role": "assistant", "content": message.content})
425
-
426
- # test multi-turn dialogue
427
- messages.append({"role": "user", "content": "express your result in json"})
428
- chat_completion = await client.chat.completions.create(
429
- model=model_name,
430
- messages=messages,
431
- max_tokens=10,
432
- )
433
- message = chat_completion.choices[0].message
434
- assert message.content is not None and len(message.content) >= 0
435
-
436
-
269
@pytest.mark.asyncio
270
@pytest.mark.parametrize(
271
"model_name",
@@ -468,126 +300,6 @@ async def test_completion_streaming(client: openai.AsyncOpenAI,
300
assert "".join(chunks) == single_output
301
302
471
-@pytest.mark.asyncio
472
-@pytest.mark.parametrize(
473
- # just test 1 lora hereafter
474
- "model_name",
475
- [MODEL_NAME, "zephyr-lora"],
476
-)
477
-async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
478
- messages = [{
479
- "role": "system",
480
- "content": "you are a helpful assistant"
481
- }, {
482
- "role": "user",
483
- "content": "what is 1+1?"
484
- }]
485
-
486
- # test single completion
487
- chat_completion = await client.chat.completions.create(
488
- model=model_name,
489
- messages=messages,
490
- max_tokens=10,
491
- temperature=0.0,
492
- )
493
- output = chat_completion.choices[0].message.content
494
- stop_reason = chat_completion.choices[0].finish_reason
495
-
496
- # test streaming
497
- stream = await client.chat.completions.create(
498
- model=model_name,
499
- messages=messages,
500
- max_tokens=10,
501
- temperature=0.0,
502
- stream=True,
503
- )
504
- chunks: List[str] = []
505
- finish_reason_count = 0
506
- async for chunk in stream:
507
- delta = chunk.choices[0].delta
508
- if delta.role:
509
- assert delta.role == "assistant"
510
- if delta.content:
511
- chunks.append(delta.content)
512
- if chunk.choices[0].finish_reason is not None:
513
- finish_reason_count += 1
514
- # finish reason should only return in last block
515
- assert finish_reason_count == 1
516
- assert chunk.choices[0].finish_reason == stop_reason
517
- assert delta.content
518
- assert "".join(chunks) == output
519
-
520
-
521
-@pytest.mark.asyncio
522
-@pytest.mark.parametrize(
523
- "model_name",
524
- ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
525
-)
526
-async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
527
- model_name: str):
528
- messages = [{
529
- "role": "system",
530
- "content": "You are a helpful assistant."
531
- }, {
532
- "role": "user",
533
- "content": "What is the capital of France?"
534
- }]
535
-
536
- # Test stream=True, stream_options={"include_usage": False}
537
- stream = await client.chat.completions.create(
538
- model=model_name,
539
- messages=messages,
540
- max_tokens=10,
541
- temperature=0.0,
542
- stream=True,
543
- stream_options={"include_usage": False})
544
- async for chunk in stream:
545
- assert chunk.usage is None
546
-
547
- # Test stream=True, stream_options={"include_usage": True}
548
- stream = await client.chat.completions.create(
549
- model=model_name,
550
- messages=messages,
551
- max_tokens=10,
552
- temperature=0.0,
553
- stream=True,
554
- stream_options={"include_usage": True})
555
-
556
- async for chunk in stream:
557
- if chunk.choices[0].finish_reason is None:
558
- assert chunk.usage is None
559
- else:
560
- assert chunk.usage is None
561
- final_chunk = await stream.__anext__()
562
- assert final_chunk.usage is not None
563
- assert final_chunk.usage.prompt_tokens > 0
564
- assert final_chunk.usage.completion_tokens > 0
565
- assert final_chunk.usage.total_tokens == (
566
- final_chunk.usage.prompt_tokens +
567
- final_chunk.usage.completion_tokens)
568
- assert final_chunk.choices == []
569
-
570
- # Test stream=False, stream_options={"include_usage": None}
571
- with pytest.raises(BadRequestError):
572
- await client.chat.completions.create(
573
- model=model_name,
574
- messages=messages,
575
- max_tokens=10,
576
- temperature=0.0,
577
- stream=False,
578
- stream_options={"include_usage": None})
579
-
580
- # Test stream=False, stream_options={"include_usage": True}
581
- with pytest.raises(BadRequestError):
582
- await client.chat.completions.create(
583
- model=model_name,
584
- messages=messages,
585
- max_tokens=10,
586
- temperature=0.0,
587
- stream=False,
588
- stream_options={"include_usage": True})
589
-
590
-
303
@pytest.mark.asyncio
304
@pytest.mark.parametrize(
305
"model_name",
@@ -773,53 +485,6 @@ async def test_guided_json_completion(client: openai.AsyncOpenAI,
485
jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
486
487
776
-@pytest.mark.asyncio
777
-@pytest.mark.parametrize("guided_decoding_backend",
778
- ["outlines", "lm-format-enforcer"])
779
-async def test_guided_json_chat(client: openai.AsyncOpenAI,
780
- guided_decoding_backend: str):
781
- messages = [{
782
- "role": "system",
783
- "content": "you are a helpful assistant"
784
- }, {
785
- "role":
786
- "user",
787
- "content":
788
- f"Give an example JSON for an employee profile that "
789
- f"fits this schema: {TEST_SCHEMA}"
790
- }]
791
- chat_completion = await client.chat.completions.create(
792
- model=MODEL_NAME,
793
- messages=messages,
794
- max_tokens=1000,
795
- extra_body=dict(guided_json=TEST_SCHEMA,
796
- guided_decoding_backend=guided_decoding_backend))
797
- message = chat_completion.choices[0].message
798
- assert message.content is not None
799
- json1 = json.loads(message.content)
800
- jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
801
-
802
- messages.append({"role": "assistant", "content": message.content})
803
- messages.append({
804
- "role":
805
- "user",
806
- "content":
807
- "Give me another one with a different name and age"
808
- })
809
- chat_completion = await client.chat.completions.create(
810
- model=MODEL_NAME,
811
- messages=messages,
812
- max_tokens=1000,
813
- extra_body=dict(guided_json=TEST_SCHEMA,
814
- guided_decoding_backend=guided_decoding_backend))
815
- message = chat_completion.choices[0].message
816
- assert message.content is not None
817
- json2 = json.loads(message.content)
818
- jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
819
- assert json1["name"] != json2["name"]
820
- assert json1["age"] != json2["age"]
821
-
822
-
488
@pytest.mark.asyncio
489
@pytest.mark.parametrize("guided_decoding_backend",
490
["outlines", "lm-format-enforcer"])
@@ -840,44 +505,6 @@ async def test_guided_regex_completion(client: openai.AsyncOpenAI,
505
assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
506
507
843
-@pytest.mark.asyncio
844
-@pytest.mark.parametrize("guided_decoding_backend",
845
- ["outlines", "lm-format-enforcer"])
846
-async def test_guided_regex_chat(client: openai.AsyncOpenAI,
847
- guided_decoding_backend: str):
848
- messages = [{
849
- "role": "system",
850
- "content": "you are a helpful assistant"
851
- }, {
852
- "role":
853
- "user",
854
- "content":
855
- f"Give an example IP address with this regex: {TEST_REGEX}"
856
- }]
857
- chat_completion = await client.chat.completions.create(
858
- model=MODEL_NAME,
859
- messages=messages,
860
- max_tokens=20,
861
- extra_body=dict(guided_regex=TEST_REGEX,
862
- guided_decoding_backend=guided_decoding_backend))
863
- ip1 = chat_completion.choices[0].message.content
864
- assert ip1 is not None
865
- assert re.fullmatch(TEST_REGEX, ip1) is not None
866
-
867
- messages.append({"role": "assistant", "content": ip1})
868
- messages.append({"role": "user", "content": "Give me a different one"})
869
- chat_completion = await client.chat.completions.create(
870
- model=MODEL_NAME,
871
- messages=messages,
872
- max_tokens=20,
873
- extra_body=dict(guided_regex=TEST_REGEX,
874
- guided_decoding_backend=guided_decoding_backend))
875
- ip2 = chat_completion.choices[0].message.content
876
- assert ip2 is not None
877
- assert re.fullmatch(TEST_REGEX, ip2) is not None
878
- assert ip1 != ip2
879
-
880
-
508
@pytest.mark.asyncio
509
@pytest.mark.parametrize("guided_decoding_backend",
510
["outlines", "lm-format-enforcer"])
@@ -898,385 +525,6 @@ async def test_guided_choice_completion(client: openai.AsyncOpenAI,
525
assert completion.choices[i].text in TEST_CHOICE
526
527
901
-@pytest.mark.asyncio
902
-@pytest.mark.parametrize("guided_decoding_backend",
903
- ["outlines", "lm-format-enforcer"])
904
-async def test_guided_choice_chat(client: openai.AsyncOpenAI,
905
- guided_decoding_backend: str):
906
- messages = [{
907
- "role": "system",
908
- "content": "you are a helpful assistant"
909
- }, {
910
- "role":
911
- "user",
912
- "content":
913
- "The best language for type-safe systems programming is "
914
- }]
915
- chat_completion = await client.chat.completions.create(
916
- model=MODEL_NAME,
917
- messages=messages,
918
- max_tokens=10,
919
- extra_body=dict(guided_choice=TEST_CHOICE,
920
- guided_decoding_backend=guided_decoding_backend))
921
- choice1 = chat_completion.choices[0].message.content
922
- assert choice1 in TEST_CHOICE
923
-
924
- messages.append({"role": "assistant", "content": choice1})
925
- messages.append({
926
- "role": "user",
927
- "content": "I disagree, pick another one"
928
- })
929
- chat_completion = await client.chat.completions.create(
930
- model=MODEL_NAME,
931
- messages=messages,
932
- max_tokens=10,
933
- extra_body=dict(guided_choice=TEST_CHOICE,
934
- guided_decoding_backend=guided_decoding_backend))
935
- choice2 = chat_completion.choices[0].message.content
936
- assert choice2 in TEST_CHOICE
937
- assert choice1 != choice2
938
-
939
-
940
-@pytest.mark.asyncio
941
-@pytest.mark.parametrize("guided_decoding_backend",
942
- ["outlines", "lm-format-enforcer"])
943
-async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
944
- guided_decoding_backend: str):
945
- with pytest.raises(openai.BadRequestError):
946
- _ = await client.completions.create(
947
- model=MODEL_NAME,
948
- prompt="Give an example JSON that fits this schema: 42",
949
- extra_body=dict(guided_json=42,
950
- guided_decoding_backend=guided_decoding_backend))
951
-
952
- messages = [{
953
- "role": "system",
954
- "content": "you are a helpful assistant"
955
- }, {
956
- "role":
957
- "user",
958
- "content":
959
- "The best language for type-safe systems programming is "
960
- }]
961
- with pytest.raises(openai.BadRequestError):
962
- _ = await client.chat.completions.create(model=MODEL_NAME,
963
- messages=messages,
964
- extra_body=dict(guided_regex={
965
- 1: "Python",
966
- 2: "C++"
967
- }))
968
-
969
- with pytest.raises(openai.BadRequestError):
970
- _ = await client.completions.create(
971
- model=MODEL_NAME,
972
- prompt="Give an example string that fits this regex",
973
- extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
974
-
975
-
976
-@pytest.mark.asyncio
977
-@pytest.mark.parametrize("guided_decoding_backend",
978
- ["outlines", "lm-format-enforcer"])
979
-async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
980
- guided_decoding_backend: str):
981
- messages = [{
982
- "role": "system",
983
- "content": "you are a helpful assistant"
984
- }, {
985
- "role":
986
- "user",
987
- "content":
988
- "The best language for type-safe systems programming is "
989
- }]
990
- chat_completion = await client.chat.completions.create(
991
- model=MODEL_NAME,
992
- messages=messages,
993
- max_tokens=10,
994
- logprobs=True,
995
- top_logprobs=5,
996
- extra_body=dict(guided_choice=TEST_CHOICE,
997
- guided_decoding_backend=guided_decoding_backend))
998
-
999
- assert chat_completion.choices[0].logprobs is not None
1000
- assert chat_completion.choices[0].logprobs.content is not None
1001
- top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
1002
-
1003
- # -9999.0 is the minimum logprob returned by OpenAI
1004
- for item in top_logprobs:
1005
- assert item.logprob >= -9999.0, f"Failed (top_logprobs={top_logprobs})"
1006
-
1007
-
1008
-@pytest.mark.asyncio
1009
-@pytest.mark.parametrize("guided_decoding_backend",
1010
- ["outlines", "lm-format-enforcer"])
1011
-async def test_named_tool_use(client: openai.AsyncOpenAI,
1012
- guided_decoding_backend: str):
1013
- messages = [{
1014
- "role": "system",
1015
- "content": "you are a helpful assistant"
1016
- }, {
1017
- "role":
1018
- "user",
1019
- "content":
1020
- f"Give an example JSON for an employee profile that "
1021
- f"fits this schema: {TEST_SCHEMA}"
1022
- }]
1023
-
1024
- # non-streaming
1025
-
1026
- chat_completion = await client.chat.completions.create(
1027
- model=MODEL_NAME,
1028
- messages=messages,
1029
- max_tokens=1000,
1030
- tools=[{
1031
- "type": "function",
1032
- "function": {
1033
- "name": "dummy_function_name",
1034
- "description": "This is a dummy function",
1035
- "parameters": TEST_SCHEMA
1036
- }
1037
- }],
1038
- tool_choice={
1039
- "type": "function",
1040
- "function": {
1041
- "name": "dummy_function_name"
1042
- }
1043
- })
1044
- message = chat_completion.choices[0].message
1045
- assert len(message.content) == 0
1046
- json_string = message.tool_calls[0].function.arguments
1047
- json1 = json.loads(json_string)
1048
- jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
1049
-
1050
- messages.append({"role": "assistant", "content": json_string})
1051
- messages.append({
1052
- "role":
1053
- "user",
1054
- "content":
1055
- "Give me another one with a different name and age"
1056
- })
1057
-
1058
- # streaming
1059
-
1060
- stream = await client.chat.completions.create(
1061
- model=MODEL_NAME,
1062
- messages=messages,
1063
- max_tokens=1000,
1064
- tools=[{
1065
- "type": "function",
1066
- "function": {
1067
- "name": "dummy_function_name",
1068
- "description": "This is a dummy function",
1069
- "parameters": TEST_SCHEMA
1070
- }
1071
- }],
1072
- tool_choice={
1073
- "type": "function",
1074
- "function": {
1075
- "name": "dummy_function_name"
1076
- }
1077
- },
1078
- stream=True)
1079
-
1080
- output = []
1081
- finish_reason_count = 0
1082
- async for chunk in stream:
1083
- delta = chunk.choices[0].delta
1084
- if delta.role:
1085
- assert delta.role == "assistant"
1086
- assert delta.content is None or len(delta.content) == 0
1087
- if delta.tool_calls:
1088
- output.append(delta.tool_calls[0].function.arguments)
1089
- if chunk.choices[0].finish_reason is not None:
1090
- finish_reason_count += 1
1091
- # finish reason should only return in last block
1092
- assert finish_reason_count == 1
1093
- json2 = json.loads("".join(output))
1094
- jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
1095
- assert json1["name"] != json2["name"]
1096
- assert json1["age"] != json2["age"]
1097
-
1098
-
1099
-@pytest.mark.asyncio
1100
-@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1101
-async def test_required_tool_use_not_yet_supported(
1102
- client: openai.AsyncOpenAI, guided_decoding_backend: str):
1103
- messages = [{
1104
- "role": "system",
1105
- "content": "you are a helpful assistant"
1106
- }, {
1107
- "role":
1108
- "user",
1109
- "content":
1110
- f"Give an example JSON for an employee profile that "
1111
- f"fits this schema: {TEST_SCHEMA}"
1112
- }]
1113
-
1114
- with pytest.raises(openai.BadRequestError):
1115
- await client.chat.completions.create(
1116
- model=MODEL_NAME,
1117
- messages=messages,
1118
- max_tokens=1000,
1119
- tools=[{
1120
- "type": "function",
1121
- "function": {
1122
- "name": "dummy_function_name",
1123
- "description": "This is a dummy function",
1124
- "parameters": TEST_SCHEMA
1125
- }
1126
- }],
1127
- tool_choice="required")
1128
-
1129
- with pytest.raises(openai.BadRequestError):
1130
- await client.chat.completions.create(
1131
- model=MODEL_NAME,
1132
- messages=messages,
1133
- max_tokens=1000,
1134
- tools=[{
1135
- "type": "function",
1136
- "function": {
1137
- "name": "dummy_function_name",
1138
- "description": "This is a dummy function",
1139
- "parameters": TEST_SCHEMA
1140
- }
1141
- }],
1142
- tool_choice="auto")
1143
-
1144
-
1145
-@pytest.mark.asyncio
1146
-@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1147
-async def test_inconsistent_tool_choice_and_tools(
1148
- client: openai.AsyncOpenAI, guided_decoding_backend: str):
1149
- messages = [{
1150
- "role": "system",
1151
- "content": "you are a helpful assistant"
1152
- }, {
1153
- "role":
1154
- "user",
1155
- "content":
1156
- f"Give an example JSON for an employee profile that "
1157
- f"fits this schema: {TEST_SCHEMA}"
1158
- }]
1159
-
1160
- with pytest.raises(openai.BadRequestError):
1161
- await client.chat.completions.create(model=MODEL_NAME,
1162
- messages=messages,
1163
- max_tokens=1000,
1164
- tool_choice={
1165
- "type": "function",
1166
- "function": {
1167
- "name":
1168
- "dummy_function_name"
1169
- }
1170
- })
1171
-
1172
- with pytest.raises(openai.BadRequestError):
1173
- await client.chat.completions.create(
1174
- model=MODEL_NAME,
1175
- messages=messages,
1176
- max_tokens=1000,
1177
- tools=[{
1178
- "type": "function",
1179
- "function": {
1180
- "name": "dummy_function_name",
1181
- "description": "This is a dummy function",
1182
- "parameters": TEST_SCHEMA
1183
- }
1184
- }],
1185
- tool_choice={
1186
- "type": "function",
1187
- "function": {
1188
- "name": "nondefined_function_name"
1189
- }
1190
- })
1191
-
1192
-
1193
-@pytest.mark.asyncio
1194
-async def test_response_format_json_object(client: openai.AsyncOpenAI):
1195
- for _ in range(2):
1196
- resp = await client.chat.completions.create(
1197
- model=MODEL_NAME,
1198
- messages=[{
1199
- "role":
1200
- "user",
1201
- "content": ('what is 1+1? please respond with a JSON object, '
1202
- 'the format is {"result": 2}')
1203
- }],
1204
- response_format={"type": "json_object"})
1205
-
1206
- content = resp.choices[0].message.content
1207
- assert content is not None
1208
-
1209
- loaded = json.loads(content)
1210
- assert loaded == {"result": 2}, loaded
1211
-
1212
-
1213
-@pytest.mark.asyncio
1214
-async def test_extra_fields(client: openai.AsyncOpenAI):
1215
- with pytest.raises(BadRequestError) as exc_info:
1216
- await client.chat.completions.create(
1217
- model=MODEL_NAME,
1218
- messages=[{
1219
- "role": "system",
1220
- "content": "You are a helpful assistant.",
1221
- "extra_field": "0",
1222
- }], # type: ignore
1223
- temperature=0,
1224
- seed=0)
1225
-
1226
- assert "extra_forbidden" in exc_info.value.message
1227
-
1228
-
1229
-@pytest.mark.asyncio
1230
-async def test_complex_message_content(client: openai.AsyncOpenAI):
1231
- resp = await client.chat.completions.create(
1232
- model=MODEL_NAME,
1233
- messages=[{
1234
- "role":
1235
- "user",
1236
- "content": [{
1237
- "type":
1238
- "text",
1239
- "text":
1240
- "what is 1+1? please provide the result without any other text."
1241
- }]
1242
- }],
1243
- temperature=0,
1244
- seed=0)
1245
- content = resp.choices[0].message.content
1246
- assert content == "2"
1247
-
1248
-
1249
-@pytest.mark.asyncio
1250
-async def test_custom_role(client: openai.AsyncOpenAI):
1251
- # Not sure how the model handles custom roles so we just check that
1252
- # both string and complex message content are handled in the same way
1253
-
1254
- resp1 = await client.chat.completions.create(
1255
- model=MODEL_NAME,
1256
- messages=[{
1257
- "role": "my-custom-role",
1258
- "content": "what is 1+1?",
1259
- }], # type: ignore
1260
- temperature=0,
1261
- seed=0)
1262
-
1263
- resp2 = await client.chat.completions.create(
1264
- model=MODEL_NAME,
1265
- messages=[{
1266
- "role": "my-custom-role",
1267
- "content": [{
1268
- "type": "text",
1269
- "text": "what is 1+1?"
1270
- }]
1271
- }], # type: ignore
1272
- temperature=0,
1273
- seed=0)
1274
-
1275
- content1 = resp1.choices[0].message.content
1276
- content2 = resp2.choices[0].message.content
1277
- assert content1 == content2
1278
-
1279
-
528
@pytest.mark.asyncio
529
async def test_guided_grammar(client: openai.AsyncOpenAI):
530
simple_sql_grammar = """
@@ -1348,23 +596,22 @@ async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
596
597
598
@pytest.mark.asyncio
1351
-async def test_long_seed(client: openai.AsyncOpenAI):
1352
- for seed in [
1353
- torch.iinfo(torch.long).min - 1,
1354
- torch.iinfo(torch.long).max + 1
1355
- ]:
1356
- with pytest.raises(BadRequestError) as exc_info:
1357
- await client.chat.completions.create(
1358
- model=MODEL_NAME,
1359
- messages=[{
1360
- "role": "system",
1361
- "content": "You are a helpful assistant.",
1362
- }],
1363
- temperature=0,
1364
- seed=seed)
1365
-
1366
- assert ("greater_than_equal" in exc_info.value.message
1367
- or "less_than_equal" in exc_info.value.message)
599
+@pytest.mark.parametrize("guided_decoding_backend",
600
+ ["outlines", "lm-format-enforcer"])
601
+async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
602
+ guided_decoding_backend: str):
603
+ with pytest.raises(openai.BadRequestError):
604
+ _ = await client.completions.create(
605
+ model=MODEL_NAME,
606
+ prompt="Give an example JSON that fits this schema: 42",
607
+ extra_body=dict(guided_json=42,
608
+ guided_decoding_backend=guided_decoding_backend))
609
+
610
+ with pytest.raises(openai.BadRequestError):
611
+ _ = await client.completions.create(
612
+ model=MODEL_NAME,
613
+ prompt="Give an example string that fits this regex",
614
+ extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
615
616
617
@pytest.mark.asyncio
@@ -1372,7 +619,7 @@ async def test_long_seed(client: openai.AsyncOpenAI):
619
"model_name",
620
[MODEL_NAME],
621
)
1375
-async def test_tokenize(server, client: openai.AsyncOpenAI, model_name: str):
622
+async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
623
base_url = str(client.base_url)[:-3]
624
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
625
@@ -1399,7 +646,7 @@ async def test_tokenize(server, client: openai.AsyncOpenAI, model_name: str):
646
"model_name",
647
[MODEL_NAME],
648
)
1402
-async def test_detokenize(server, client: openai.AsyncOpenAI, model_name: str):
649
+async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
650
base_url = str(client.base_url)[:-3]
651
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
652
tests/entrypoints/test_openai_vision.py
+2
-2
@@ -1,5 +1,5 @@
1
from pathlib import Path
2
-from typing import Dict
2
+from typing import Dict, List
3
4
import openai
5
import pytest
@@ -216,7 +216,7 @@ async def test_chat_streaming_image(client: openai.AsyncOpenAI,
216
temperature=0.0,
217
stream=True,
218
)
219
- chunks = []
219
+ chunks: List[str] = []
220
finish_reason_count = 0
221
async for chunk in stream:
222
delta = chunk.choices[0].delta