@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e9d32d077d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[CI/Build] [1/3] Reorganize entrypoints tests (#5526)

Cyrus Leung committed Jun 27, 2024 at 20:43 UTC e9d32d077da2f914c965a66dd2fdfc77c50d117f
3 files changed +896 -774
tests/entrypoints/test_openai_chat.py new
+875
@@ -0,0 +1,875 @@
1 +# imports for guided decoding tests
2 +import json
3 +import re
4 +from typing import List
5 +
6 +import jsonschema
7 +import openai # use the official client for correctness check
8 +import pytest
9 +# using Ray for overall ease of process management, parallel requests,
10 +# and debugging.
11 +import ray
12 +import torch
13 +# downloading lora to test lora requests
14 +from huggingface_hub import snapshot_download
15 +from openai import BadRequestError
16 +
17 +from ..utils import VLLM_PATH, RemoteOpenAIServer
18 +
19 +# any model with a chat template should work here
20 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
21 +# technically this needs Mistral-7B-v0.1 as base, but we're not testing
22 +# generation quality here
23 +LORA_NAME = "typeof/zephyr-7b-beta-lora"
24 +
25 +TEST_SCHEMA = {
26 + "type": "object",
27 + "properties": {
28 + "name": {
29 + "type": "string"
30 + },
31 + "age": {
32 + "type": "integer"
33 + },
34 + "skills": {
35 + "type": "array",
36 + "items": {
37 + "type": "string",
38 + "maxLength": 10
39 + },
40 + "minItems": 3
41 + },
42 + "work history": {
43 + "type": "array",
44 + "items": {
45 + "type": "object",
46 + "properties": {
47 + "company": {
48 + "type": "string"
49 + },
50 + "duration": {
51 + "type": "string"
52 + },
53 + "position": {
54 + "type": "string"
55 + }
56 + },
57 + "required": ["company", "position"]
58 + }
59 + }
60 + },
61 + "required": ["name", "age", "skills", "work history"]
62 +}
63 +
64 +TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
65 + r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
66 +
67 +TEST_CHOICE = [
68 + "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
69 + "Swift", "Kotlin"
70 +]
71 +
72 +pytestmark = pytest.mark.openai
73 +
74 +
75 +@pytest.fixture(scope="session")
76 +def zephyr_lora_files():
77 + return snapshot_download(repo_id=LORA_NAME)
78 +
79 +
80 +@pytest.fixture(scope="module")
81 +def ray_ctx():
82 + ray.init(runtime_env={"working_dir": VLLM_PATH})
83 + yield
84 + ray.shutdown()
85 +
86 +
87 +@pytest.fixture(scope="module")
88 +def server(zephyr_lora_files, ray_ctx):
89 + return RemoteOpenAIServer([
90 + "--model",
91 + MODEL_NAME,
92 + # use half precision for speed and memory savings in CI environment
93 + "--dtype",
94 + "bfloat16",
95 + "--max-model-len",
96 + "8192",
97 + "--enforce-eager",
98 + # lora config below
99 + "--enable-lora",
100 + "--lora-modules",
101 + f"zephyr-lora={zephyr_lora_files}",
102 + f"zephyr-lora2={zephyr_lora_files}",
103 + "--max-lora-rank",
104 + "64",
105 + "--max-cpu-loras",
106 + "2",
107 + "--max-num-seqs",
108 + "128",
109 + ])
110 +
111 +
112 +@pytest.fixture(scope="module")
113 +def client(server):
114 + return server.get_async_client()
115 +
116 +
117 +@pytest.mark.asyncio
118 +@pytest.mark.parametrize(
119 + # first test base model, then test loras
120 + "model_name",
121 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
122 +)
123 +async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
124 + messages = [{
125 + "role": "system",
126 + "content": "you are a helpful assistant"
127 + }, {
128 + "role": "user",
129 + "content": "what is 1+1?"
130 + }]
131 +
132 + chat_completion = await client.chat.completions.create(model=model_name,
133 + messages=messages,
134 + max_tokens=5,
135 + temperature=0.0,
136 + logprobs=False)
137 +
138 + choice = chat_completion.choices[0]
139 + assert choice.logprobs is None
140 +
141 +
142 +@pytest.mark.asyncio
143 +@pytest.mark.parametrize(
144 + # just test 1 lora hereafter
145 + "model_name",
146 + [MODEL_NAME, "zephyr-lora"],
147 +)
148 +async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
149 + messages = [{
150 + "role": "system",
151 + "content": "you are a helpful assistant"
152 + }, {
153 + "role": "user",
154 + "content": "what is 1+1?"
155 + }]
156 +
157 + chat_completion = await client.chat.completions.create(model=model_name,
158 + messages=messages,
159 + max_tokens=5,
160 + temperature=0.0,
161 + logprobs=True,
162 + top_logprobs=0)
163 +
164 + choice = chat_completion.choices[0]
165 + assert choice.logprobs is not None
166 + assert choice.logprobs.content is not None
167 + assert len(choice.logprobs.content[0].top_logprobs) == 0
168 +
169 +
170 +@pytest.mark.asyncio
171 +@pytest.mark.parametrize(
172 + "model_name",
173 + [MODEL_NAME, "zephyr-lora"],
174 +)
175 +async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
176 + messages = [{
177 + "role": "system",
178 + "content": "you are a helpful assistant"
179 + }, {
180 + "role": "user",
181 + "content": "what is 1+1?"
182 + }]
183 +
184 + chat_completion = await client.chat.completions.create(model=model_name,
185 + messages=messages,
186 + max_tokens=5,
187 + temperature=0.0,
188 + logprobs=True,
189 + top_logprobs=5)
190 +
191 + choice = chat_completion.choices[0]
192 + assert choice.logprobs is not None
193 + assert choice.logprobs.content is not None
194 + assert len(choice.logprobs.content[0].top_logprobs) == 5
195 +
196 +
197 +@pytest.mark.asyncio
198 +@pytest.mark.parametrize(
199 + "model_name",
200 + [MODEL_NAME, "zephyr-lora"],
201 +)
202 +async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
203 + model_name: str):
204 + messages = [{
205 + "role": "system",
206 + "content": "you are a helpful assistant"
207 + }, {
208 + "role": "user",
209 + "content": "what is 1+1?"
210 + }]
211 +
212 + # Default max_logprobs is 20, so this should raise an error
213 + with pytest.raises((openai.BadRequestError, openai.APIError)):
214 + stream = await client.chat.completions.create(model=model_name,
215 + messages=messages,
216 + max_tokens=10,
217 + logprobs=True,
218 + top_logprobs=21,
219 + stream=True)
220 + async for chunk in stream:
221 + ...
222 +
223 + with pytest.raises(openai.BadRequestError):
224 + await client.chat.completions.create(model=model_name,
225 + messages=messages,
226 + max_tokens=10,
227 + logprobs=True,
228 + top_logprobs=30,
229 + stream=False)
230 +
231 + # the server should still work afterwards
232 + chat_completion = await client.chat.completions.create(model=model_name,
233 + messages=messages,
234 + max_tokens=10,
235 + stream=False)
236 + message = chat_completion.choices[0].message
237 + assert message.content is not None and len(message.content) >= 0
238 +
239 +
240 +@pytest.mark.asyncio
241 +@pytest.mark.parametrize(
242 + "model_name",
243 + [MODEL_NAME, "zephyr-lora"],
244 +)
245 +async def test_single_chat_session(client: openai.AsyncOpenAI,
246 + model_name: str):
247 + messages = [{
248 + "role": "system",
249 + "content": "you are a helpful assistant"
250 + }, {
251 + "role": "user",
252 + "content": "what is 1+1?"
253 + }]
254 +
255 + # test single completion
256 + chat_completion = await client.chat.completions.create(model=model_name,
257 + messages=messages,
258 + max_tokens=10,
259 + logprobs=True,
260 + top_logprobs=5)
261 + assert chat_completion.id is not None
262 + assert len(chat_completion.choices) == 1
263 +
264 + choice = chat_completion.choices[0]
265 + assert choice.finish_reason == "length"
266 + assert chat_completion.usage == openai.types.CompletionUsage(
267 + completion_tokens=10, prompt_tokens=37, total_tokens=47)
268 +
269 + message = choice.message
270 + assert message.content is not None and len(message.content) >= 10
271 + assert message.role == "assistant"
272 + messages.append({"role": "assistant", "content": message.content})
273 +
274 + # test multi-turn dialogue
275 + messages.append({"role": "user", "content": "express your result in json"})
276 + chat_completion = await client.chat.completions.create(
277 + model=model_name,
278 + messages=messages,
279 + max_tokens=10,
280 + )
281 + message = chat_completion.choices[0].message
282 + assert message.content is not None and len(message.content) >= 0
283 +
284 +
285 +@pytest.mark.asyncio
286 +@pytest.mark.parametrize(
287 + # just test 1 lora hereafter
288 + "model_name",
289 + [MODEL_NAME, "zephyr-lora"],
290 +)
291 +async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
292 + messages = [{
293 + "role": "system",
294 + "content": "you are a helpful assistant"
295 + }, {
296 + "role": "user",
297 + "content": "what is 1+1?"
298 + }]
299 +
300 + # test single completion
301 + chat_completion = await client.chat.completions.create(
302 + model=model_name,
303 + messages=messages,
304 + max_tokens=10,
305 + temperature=0.0,
306 + )
307 + output = chat_completion.choices[0].message.content
308 + stop_reason = chat_completion.choices[0].finish_reason
309 +
310 + # test streaming
311 + stream = await client.chat.completions.create(
312 + model=model_name,
313 + messages=messages,
314 + max_tokens=10,
315 + temperature=0.0,
316 + stream=True,
317 + )
318 + chunks: List[str] = []
319 + finish_reason_count = 0
320 + async for chunk in stream:
321 + delta = chunk.choices[0].delta
322 + if delta.role:
323 + assert delta.role == "assistant"
324 + if delta.content:
325 + chunks.append(delta.content)
326 + if chunk.choices[0].finish_reason is not None:
327 + finish_reason_count += 1
328 + # finish reason should only return in last block
329 + assert finish_reason_count == 1
330 + assert chunk.choices[0].finish_reason == stop_reason
331 + assert delta.content
332 + assert "".join(chunks) == output
333 +
334 +
335 +@pytest.mark.asyncio
336 +@pytest.mark.parametrize(
337 + "model_name",
338 + ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
339 +)
340 +async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
341 + model_name: str):
342 + messages = [{
343 + "role": "system",
344 + "content": "You are a helpful assistant."
345 + }, {
346 + "role": "user",
347 + "content": "What is the capital of France?"
348 + }]
349 +
350 + # Test stream=True, stream_options={"include_usage": False}
351 + stream = await client.chat.completions.create(
352 + model=model_name,
353 + messages=messages,
354 + max_tokens=10,
355 + temperature=0.0,
356 + stream=True,
357 + stream_options={"include_usage": False})
358 + async for chunk in stream:
359 + assert chunk.usage is None
360 +
361 + # Test stream=True, stream_options={"include_usage": True}
362 + stream = await client.chat.completions.create(
363 + model=model_name,
364 + messages=messages,
365 + max_tokens=10,
366 + temperature=0.0,
367 + stream=True,
368 + stream_options={"include_usage": True})
369 +
370 + async for chunk in stream:
371 + if chunk.choices[0].finish_reason is None:
372 + assert chunk.usage is None
373 + else:
374 + assert chunk.usage is None
375 + final_chunk = await stream.__anext__()
376 + assert final_chunk.usage is not None
377 + assert final_chunk.usage.prompt_tokens > 0
378 + assert final_chunk.usage.completion_tokens > 0
379 + assert final_chunk.usage.total_tokens == (
380 + final_chunk.usage.prompt_tokens +
381 + final_chunk.usage.completion_tokens)
382 + assert final_chunk.choices == []
383 +
384 + # Test stream=False, stream_options={"include_usage": None}
385 + with pytest.raises(BadRequestError):
386 + await client.chat.completions.create(
387 + model=model_name,
388 + messages=messages,
389 + max_tokens=10,
390 + temperature=0.0,
391 + stream=False,
392 + stream_options={"include_usage": None})
393 +
394 + # Test stream=False, stream_options={"include_usage": True}
395 + with pytest.raises(BadRequestError):
396 + await client.chat.completions.create(
397 + model=model_name,
398 + messages=messages,
399 + max_tokens=10,
400 + temperature=0.0,
401 + stream=False,
402 + stream_options={"include_usage": True})
403 +
404 +
405 +# NOTE: Not sure why, but when I place this after `test_guided_regex_chat`
406 +# (i.e. using the same ordering as in the Completions API tests), the test
407 +# will fail on the second `guided_decoding_backend` even when I swap their order
408 +# (ref: https://github.com/vllm-project/vllm/pull/5526#issuecomment-2173772256)
409 +@pytest.mark.asyncio
410 +@pytest.mark.parametrize("guided_decoding_backend",
411 + ["outlines", "lm-format-enforcer"])
412 +async def test_guided_choice_chat(client: openai.AsyncOpenAI,
413 + guided_decoding_backend: str):
414 + messages = [{
415 + "role": "system",
416 + "content": "you are a helpful assistant"
417 + }, {
418 + "role":
419 + "user",
420 + "content":
421 + "The best language for type-safe systems programming is "
422 + }]
423 + chat_completion = await client.chat.completions.create(
424 + model=MODEL_NAME,
425 + messages=messages,
426 + max_tokens=10,
427 + extra_body=dict(guided_choice=TEST_CHOICE,
428 + guided_decoding_backend=guided_decoding_backend))
429 + choice1 = chat_completion.choices[0].message.content
430 + assert choice1 in TEST_CHOICE
431 +
432 + messages.append({"role": "assistant", "content": choice1})
433 + messages.append({
434 + "role": "user",
435 + "content": "I disagree, pick another one"
436 + })
437 + chat_completion = await client.chat.completions.create(
438 + model=MODEL_NAME,
439 + messages=messages,
440 + max_tokens=10,
441 + extra_body=dict(guided_choice=TEST_CHOICE,
442 + guided_decoding_backend=guided_decoding_backend))
443 + choice2 = chat_completion.choices[0].message.content
444 + assert choice2 in TEST_CHOICE
445 + assert choice1 != choice2
446 +
447 +
448 +@pytest.mark.asyncio
449 +@pytest.mark.parametrize("guided_decoding_backend",
450 + ["outlines", "lm-format-enforcer"])
451 +async def test_guided_json_chat(client: openai.AsyncOpenAI,
452 + guided_decoding_backend: str):
453 + messages = [{
454 + "role": "system",
455 + "content": "you are a helpful assistant"
456 + }, {
457 + "role":
458 + "user",
459 + "content":
460 + f"Give an example JSON for an employee profile that "
461 + f"fits this schema: {TEST_SCHEMA}"
462 + }]
463 + chat_completion = await client.chat.completions.create(
464 + model=MODEL_NAME,
465 + messages=messages,
466 + max_tokens=1000,
467 + extra_body=dict(guided_json=TEST_SCHEMA,
468 + guided_decoding_backend=guided_decoding_backend))
469 + message = chat_completion.choices[0].message
470 + assert message.content is not None
471 + json1 = json.loads(message.content)
472 + jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
473 +
474 + messages.append({"role": "assistant", "content": message.content})
475 + messages.append({
476 + "role":
477 + "user",
478 + "content":
479 + "Give me another one with a different name and age"
480 + })
481 + chat_completion = await client.chat.completions.create(
482 + model=MODEL_NAME,
483 + messages=messages,
484 + max_tokens=1000,
485 + extra_body=dict(guided_json=TEST_SCHEMA,
486 + guided_decoding_backend=guided_decoding_backend))
487 + message = chat_completion.choices[0].message
488 + assert message.content is not None
489 + json2 = json.loads(message.content)
490 + jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
491 + assert json1["name"] != json2["name"]
492 + assert json1["age"] != json2["age"]
493 +
494 +
495 +@pytest.mark.asyncio
496 +@pytest.mark.parametrize("guided_decoding_backend",
497 + ["outlines", "lm-format-enforcer"])
498 +async def test_guided_regex_chat(client: openai.AsyncOpenAI,
499 + guided_decoding_backend: str):
500 + messages = [{
501 + "role": "system",
502 + "content": "you are a helpful assistant"
503 + }, {
504 + "role":
505 + "user",
506 + "content":
507 + f"Give an example IP address with this regex: {TEST_REGEX}"
508 + }]
509 + chat_completion = await client.chat.completions.create(
510 + model=MODEL_NAME,
511 + messages=messages,
512 + max_tokens=20,
513 + extra_body=dict(guided_regex=TEST_REGEX,
514 + guided_decoding_backend=guided_decoding_backend))
515 + ip1 = chat_completion.choices[0].message.content
516 + assert ip1 is not None
517 + assert re.fullmatch(TEST_REGEX, ip1) is not None
518 +
519 + messages.append({"role": "assistant", "content": ip1})
520 + messages.append({"role": "user", "content": "Give me a different one"})
521 + chat_completion = await client.chat.completions.create(
522 + model=MODEL_NAME,
523 + messages=messages,
524 + max_tokens=20,
525 + extra_body=dict(guided_regex=TEST_REGEX,
526 + guided_decoding_backend=guided_decoding_backend))
527 + ip2 = chat_completion.choices[0].message.content
528 + assert ip2 is not None
529 + assert re.fullmatch(TEST_REGEX, ip2) is not None
530 + assert ip1 != ip2
531 +
532 +
533 +@pytest.mark.asyncio
534 +async def test_guided_decoding_type_error(client: openai.AsyncOpenAI):
535 + messages = [{
536 + "role": "system",
537 + "content": "you are a helpful assistant"
538 + }, {
539 + "role":
540 + "user",
541 + "content":
542 + "The best language for type-safe systems programming is "
543 + }]
544 +
545 + with pytest.raises(openai.BadRequestError):
546 + _ = await client.chat.completions.create(model=MODEL_NAME,
547 + messages=messages,
548 + extra_body=dict(guided_regex={
549 + 1: "Python",
550 + 2: "C++"
551 + }))
552 +
553 +
554 +@pytest.mark.asyncio
555 +@pytest.mark.parametrize("guided_decoding_backend",
556 + ["outlines", "lm-format-enforcer"])
557 +async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
558 + guided_decoding_backend: str):
559 + messages = [{
560 + "role": "system",
561 + "content": "you are a helpful assistant"
562 + }, {
563 + "role":
564 + "user",
565 + "content":
566 + "The best language for type-safe systems programming is "
567 + }]
568 + chat_completion = await client.chat.completions.create(
569 + model=MODEL_NAME,
570 + messages=messages,
571 + max_tokens=10,
572 + logprobs=True,
573 + top_logprobs=5,
574 + extra_body=dict(guided_choice=TEST_CHOICE,
575 + guided_decoding_backend=guided_decoding_backend))
576 +
577 + assert chat_completion.choices[0].logprobs is not None
578 + assert chat_completion.choices[0].logprobs.content is not None
579 + top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
580 +
581 + # -9999.0 is the minimum logprob returned by OpenAI
582 + for item in top_logprobs:
583 + assert item.logprob >= -9999.0, f"Failed (top_logprobs={top_logprobs})"
584 +
585 +
586 +@pytest.mark.asyncio
587 +@pytest.mark.parametrize("guided_decoding_backend",
588 + ["outlines", "lm-format-enforcer"])
589 +async def test_named_tool_use(client: openai.AsyncOpenAI,
590 + guided_decoding_backend: str):
591 + messages = [{
592 + "role": "system",
593 + "content": "you are a helpful assistant"
594 + }, {
595 + "role":
596 + "user",
597 + "content":
598 + f"Give an example JSON for an employee profile that "
599 + f"fits this schema: {TEST_SCHEMA}"
600 + }]
601 +
602 + # non-streaming
603 +
604 + chat_completion = await client.chat.completions.create(
605 + model=MODEL_NAME,
606 + messages=messages,
607 + max_tokens=1000,
608 + tools=[{
609 + "type": "function",
610 + "function": {
611 + "name": "dummy_function_name",
612 + "description": "This is a dummy function",
613 + "parameters": TEST_SCHEMA
614 + }
615 + }],
616 + tool_choice={
617 + "type": "function",
618 + "function": {
619 + "name": "dummy_function_name"
620 + }
621 + })
622 + message = chat_completion.choices[0].message
623 + assert len(message.content) == 0
624 + json_string = message.tool_calls[0].function.arguments
625 + json1 = json.loads(json_string)
626 + jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
627 +
628 + messages.append({"role": "assistant", "content": json_string})
629 + messages.append({
630 + "role":
631 + "user",
632 + "content":
633 + "Give me another one with a different name and age"
634 + })
635 +
636 + # streaming
637 +
638 + stream = await client.chat.completions.create(
639 + model=MODEL_NAME,
640 + messages=messages,
641 + max_tokens=1000,
642 + tools=[{
643 + "type": "function",
644 + "function": {
645 + "name": "dummy_function_name",
646 + "description": "This is a dummy function",
647 + "parameters": TEST_SCHEMA
648 + }
649 + }],
650 + tool_choice={
651 + "type": "function",
652 + "function": {
653 + "name": "dummy_function_name"
654 + }
655 + },
656 + stream=True)
657 +
658 + output = []
659 + finish_reason_count = 0
660 + async for chunk in stream:
661 + delta = chunk.choices[0].delta
662 + if delta.role:
663 + assert delta.role == "assistant"
664 + assert delta.content is None or len(delta.content) == 0
665 + if delta.tool_calls:
666 + output.append(delta.tool_calls[0].function.arguments)
667 + if chunk.choices[0].finish_reason is not None:
668 + finish_reason_count += 1
669 + # finish reason should only return in last block
670 + assert finish_reason_count == 1
671 + json2 = json.loads("".join(output))
672 + jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
673 + assert json1["name"] != json2["name"]
674 + assert json1["age"] != json2["age"]
675 +
676 +
677 +@pytest.mark.asyncio
678 +@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
679 +async def test_required_tool_use_not_yet_supported(
680 + client: openai.AsyncOpenAI, guided_decoding_backend: str):
681 + messages = [{
682 + "role": "system",
683 + "content": "you are a helpful assistant"
684 + }, {
685 + "role":
686 + "user",
687 + "content":
688 + f"Give an example JSON for an employee profile that "
689 + f"fits this schema: {TEST_SCHEMA}"
690 + }]
691 +
692 + with pytest.raises(openai.BadRequestError):
693 + await client.chat.completions.create(
694 + model=MODEL_NAME,
695 + messages=messages,
696 + max_tokens=1000,
697 + tools=[{
698 + "type": "function",
699 + "function": {
700 + "name": "dummy_function_name",
701 + "description": "This is a dummy function",
702 + "parameters": TEST_SCHEMA
703 + }
704 + }],
705 + tool_choice="required")
706 +
707 + with pytest.raises(openai.BadRequestError):
708 + await client.chat.completions.create(
709 + model=MODEL_NAME,
710 + messages=messages,
711 + max_tokens=1000,
712 + tools=[{
713 + "type": "function",
714 + "function": {
715 + "name": "dummy_function_name",
716 + "description": "This is a dummy function",
717 + "parameters": TEST_SCHEMA
718 + }
719 + }],
720 + tool_choice="auto")
721 +
722 +
723 +@pytest.mark.asyncio
724 +@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
725 +async def test_inconsistent_tool_choice_and_tools(
726 + client: openai.AsyncOpenAI, guided_decoding_backend: str):
727 + messages = [{
728 + "role": "system",
729 + "content": "you are a helpful assistant"
730 + }, {
731 + "role":
732 + "user",
733 + "content":
734 + f"Give an example JSON for an employee profile that "
735 + f"fits this schema: {TEST_SCHEMA}"
736 + }]
737 +
738 + with pytest.raises(openai.BadRequestError):
739 + await client.chat.completions.create(model=MODEL_NAME,
740 + messages=messages,
741 + max_tokens=1000,
742 + tool_choice={
743 + "type": "function",
744 + "function": {
745 + "name":
746 + "dummy_function_name"
747 + }
748 + })
749 +
750 + with pytest.raises(openai.BadRequestError):
751 + await client.chat.completions.create(
752 + model=MODEL_NAME,
753 + messages=messages,
754 + max_tokens=1000,
755 + tools=[{
756 + "type": "function",
757 + "function": {
758 + "name": "dummy_function_name",
759 + "description": "This is a dummy function",
760 + "parameters": TEST_SCHEMA
761 + }
762 + }],
763 + tool_choice={
764 + "type": "function",
765 + "function": {
766 + "name": "nondefined_function_name"
767 + }
768 + })
769 +
770 +
771 +@pytest.mark.asyncio
772 +async def test_response_format_json_object(client: openai.AsyncOpenAI):
773 + for _ in range(2):
774 + resp = await client.chat.completions.create(
775 + model=MODEL_NAME,
776 + messages=[{
777 + "role":
778 + "user",
779 + "content": ('what is 1+1? please respond with a JSON object, '
780 + 'the format is {"result": 2}')
781 + }],
782 + response_format={"type": "json_object"})
783 +
784 + content = resp.choices[0].message.content
785 + assert content is not None
786 +
787 + loaded = json.loads(content)
788 + assert loaded == {"result": 2}, loaded
789 +
790 +
791 +@pytest.mark.asyncio
792 +async def test_extra_fields(client: openai.AsyncOpenAI):
793 + with pytest.raises(BadRequestError) as exc_info:
794 + await client.chat.completions.create(
795 + model=MODEL_NAME,
796 + messages=[{
797 + "role": "system",
798 + "content": "You are a helpful assistant.",
799 + "extra_field": "0",
800 + }], # type: ignore
801 + temperature=0,
802 + seed=0)
803 +
804 + assert "extra_forbidden" in exc_info.value.message
805 +
806 +
807 +@pytest.mark.asyncio
808 +async def test_complex_message_content(client: openai.AsyncOpenAI):
809 + resp = await client.chat.completions.create(
810 + model=MODEL_NAME,
811 + messages=[{
812 + "role":
813 + "user",
814 + "content": [{
815 + "type":
816 + "text",
817 + "text":
818 + "what is 1+1? please provide the result without any other text."
819 + }]
820 + }],
821 + temperature=0,
822 + seed=0)
823 + content = resp.choices[0].message.content
824 + assert content == "2"
825 +
826 +
827 +@pytest.mark.asyncio
828 +async def test_custom_role(client: openai.AsyncOpenAI):
829 + # Not sure how the model handles custom roles so we just check that
830 + # both string and complex message content are handled in the same way
831 +
832 + resp1 = await client.chat.completions.create(
833 + model=MODEL_NAME,
834 + messages=[{
835 + "role": "my-custom-role",
836 + "content": "what is 1+1?",
837 + }], # type: ignore
838 + temperature=0,
839 + seed=0)
840 +
841 + resp2 = await client.chat.completions.create(
842 + model=MODEL_NAME,
843 + messages=[{
844 + "role": "my-custom-role",
845 + "content": [{
846 + "type": "text",
847 + "text": "what is 1+1?"
848 + }]
849 + }], # type: ignore
850 + temperature=0,
851 + seed=0)
852 +
853 + content1 = resp1.choices[0].message.content
854 + content2 = resp2.choices[0].message.content
855 + assert content1 == content2
856 +
857 +
858 +@pytest.mark.asyncio
859 +async def test_long_seed(client: openai.AsyncOpenAI):
860 + for seed in [
861 + torch.iinfo(torch.long).min - 1,
862 + torch.iinfo(torch.long).max + 1
863 + ]:
864 + with pytest.raises(BadRequestError) as exc_info:
865 + await client.chat.completions.create(
866 + model=MODEL_NAME,
867 + messages=[{
868 + "role": "system",
869 + "content": "You are a helpful assistant.",
870 + }],
871 + temperature=0,
872 + seed=seed)
873 +
874 + assert ("greater_than_equal" in exc_info.value.message
875 + or "less_than_equal" in exc_info.value.message)
tests/entrypoints/test_openai_server.py
+19 -772
@@ -10,7 +10,6 @@ import pytest
10 # and debugging.
11 import ray
12 import requests
13 -import torch
13 # downloading lora to test lora requests
14 from huggingface_hub import snapshot_download
15 from openai import BadRequestError
@@ -117,6 +116,7 @@ def client(server):
116 return server.get_async_client()
117
118
119 +@pytest.mark.asyncio
120 async def test_check_models(client: openai.AsyncOpenAI):
121 models = await client.models.list()
122 models = models.data
@@ -266,174 +266,6 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
266 assert len(completion.choices[0].text) >= 0
267
268
269 -@pytest.mark.asyncio
270 -@pytest.mark.parametrize(
271 - # first test base model, then test loras
272 - "model_name",
273 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
274 -)
275 -async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
276 - messages = [{
277 - "role": "system",
278 - "content": "you are a helpful assistant"
279 - }, {
280 - "role": "user",
281 - "content": "what is 1+1?"
282 - }]
283 -
284 - chat_completion = await client.chat.completions.create(model=model_name,
285 - messages=messages,
286 - max_tokens=5,
287 - temperature=0.0,
288 - logprobs=False)
289 -
290 - choice = chat_completion.choices[0]
291 - assert choice.logprobs is None
292 -
293 -
294 -@pytest.mark.asyncio
295 -@pytest.mark.parametrize(
296 - # just test 1 lora hereafter
297 - "model_name",
298 - [MODEL_NAME, "zephyr-lora"],
299 -)
300 -async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
301 - messages = [{
302 - "role": "system",
303 - "content": "you are a helpful assistant"
304 - }, {
305 - "role": "user",
306 - "content": "what is 1+1?"
307 - }]
308 -
309 - chat_completion = await client.chat.completions.create(model=model_name,
310 - messages=messages,
311 - max_tokens=5,
312 - temperature=0.0,
313 - logprobs=True,
314 - top_logprobs=0)
315 -
316 - choice = chat_completion.choices[0]
317 - assert choice.logprobs is not None
318 - assert choice.logprobs.content is not None
319 - assert len(choice.logprobs.content[0].top_logprobs) == 0
320 -
321 -
322 -@pytest.mark.asyncio
323 -@pytest.mark.parametrize(
324 - "model_name",
325 - [MODEL_NAME, "zephyr-lora"],
326 -)
327 -async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
328 - messages = [{
329 - "role": "system",
330 - "content": "you are a helpful assistant"
331 - }, {
332 - "role": "user",
333 - "content": "what is 1+1?"
334 - }]
335 -
336 - chat_completion = await client.chat.completions.create(model=model_name,
337 - messages=messages,
338 - max_tokens=5,
339 - temperature=0.0,
340 - logprobs=True,
341 - top_logprobs=5)
342 -
343 - choice = chat_completion.choices[0]
344 - assert choice.logprobs is not None
345 - assert choice.logprobs.content is not None
346 - assert len(choice.logprobs.content[0].top_logprobs) == 5
347 -
348 -
349 -@pytest.mark.asyncio
350 -@pytest.mark.parametrize(
351 - "model_name",
352 - [MODEL_NAME, "zephyr-lora"],
353 -)
354 -async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
355 - model_name: str):
356 - messages = [{
357 - "role": "system",
358 - "content": "you are a helpful assistant"
359 - }, {
360 - "role": "user",
361 - "content": "what is 1+1?"
362 - }]
363 -
364 - # Default max_logprobs is 20, so this should raise an error
365 - with pytest.raises((openai.BadRequestError, openai.APIError)):
366 - stream = await client.chat.completions.create(model=model_name,
367 - messages=messages,
368 - max_tokens=10,
369 - logprobs=True,
370 - top_logprobs=21,
371 - stream=True)
372 - async for chunk in stream:
373 - ...
374 -
375 - with pytest.raises(openai.BadRequestError):
376 - await client.chat.completions.create(model=model_name,
377 - messages=messages,
378 - max_tokens=10,
379 - logprobs=True,
380 - top_logprobs=30,
381 - stream=False)
382 -
383 - # the server should still work afterwards
384 - chat_completion = await client.chat.completions.create(model=model_name,
385 - messages=messages,
386 - max_tokens=10,
387 - stream=False)
388 - message = chat_completion.choices[0].message
389 - assert message.content is not None and len(message.content) >= 0
390 -
391 -
392 -@pytest.mark.asyncio
393 -@pytest.mark.parametrize(
394 - "model_name",
395 - [MODEL_NAME, "zephyr-lora"],
396 -)
397 -async def test_single_chat_session(client: openai.AsyncOpenAI,
398 - model_name: str):
399 - messages = [{
400 - "role": "system",
401 - "content": "you are a helpful assistant"
402 - }, {
403 - "role": "user",
404 - "content": "what is 1+1?"
405 - }]
406 -
407 - # test single completion
408 - chat_completion = await client.chat.completions.create(model=model_name,
409 - messages=messages,
410 - max_tokens=10,
411 - logprobs=True,
412 - top_logprobs=5)
413 - assert chat_completion.id is not None
414 - assert len(chat_completion.choices) == 1
415 -
416 - choice = chat_completion.choices[0]
417 - assert choice.finish_reason == "length"
418 - assert chat_completion.usage == openai.types.CompletionUsage(
419 - completion_tokens=10, prompt_tokens=37, total_tokens=47)
420 -
421 - message = choice.message
422 - assert message.content is not None and len(message.content) >= 10
423 - assert message.role == "assistant"
424 - messages.append({"role": "assistant", "content": message.content})
425 -
426 - # test multi-turn dialogue
427 - messages.append({"role": "user", "content": "express your result in json"})
428 - chat_completion = await client.chat.completions.create(
429 - model=model_name,
430 - messages=messages,
431 - max_tokens=10,
432 - )
433 - message = chat_completion.choices[0].message
434 - assert message.content is not None and len(message.content) >= 0
435 -
436 -
269 @pytest.mark.asyncio
270 @pytest.mark.parametrize(
271 "model_name",
@@ -468,126 +300,6 @@ async def test_completion_streaming(client: openai.AsyncOpenAI,
300 assert "".join(chunks) == single_output
301
302
471 -@pytest.mark.asyncio
472 -@pytest.mark.parametrize(
473 - # just test 1 lora hereafter
474 - "model_name",
475 - [MODEL_NAME, "zephyr-lora"],
476 -)
477 -async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
478 - messages = [{
479 - "role": "system",
480 - "content": "you are a helpful assistant"
481 - }, {
482 - "role": "user",
483 - "content": "what is 1+1?"
484 - }]
485 -
486 - # test single completion
487 - chat_completion = await client.chat.completions.create(
488 - model=model_name,
489 - messages=messages,
490 - max_tokens=10,
491 - temperature=0.0,
492 - )
493 - output = chat_completion.choices[0].message.content
494 - stop_reason = chat_completion.choices[0].finish_reason
495 -
496 - # test streaming
497 - stream = await client.chat.completions.create(
498 - model=model_name,
499 - messages=messages,
500 - max_tokens=10,
501 - temperature=0.0,
502 - stream=True,
503 - )
504 - chunks: List[str] = []
505 - finish_reason_count = 0
506 - async for chunk in stream:
507 - delta = chunk.choices[0].delta
508 - if delta.role:
509 - assert delta.role == "assistant"
510 - if delta.content:
511 - chunks.append(delta.content)
512 - if chunk.choices[0].finish_reason is not None:
513 - finish_reason_count += 1
514 - # finish reason should only return in last block
515 - assert finish_reason_count == 1
516 - assert chunk.choices[0].finish_reason == stop_reason
517 - assert delta.content
518 - assert "".join(chunks) == output
519 -
520 -
521 -@pytest.mark.asyncio
522 -@pytest.mark.parametrize(
523 - "model_name",
524 - ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
525 -)
526 -async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
527 - model_name: str):
528 - messages = [{
529 - "role": "system",
530 - "content": "You are a helpful assistant."
531 - }, {
532 - "role": "user",
533 - "content": "What is the capital of France?"
534 - }]
535 -
536 - # Test stream=True, stream_options={"include_usage": False}
537 - stream = await client.chat.completions.create(
538 - model=model_name,
539 - messages=messages,
540 - max_tokens=10,
541 - temperature=0.0,
542 - stream=True,
543 - stream_options={"include_usage": False})
544 - async for chunk in stream:
545 - assert chunk.usage is None
546 -
547 - # Test stream=True, stream_options={"include_usage": True}
548 - stream = await client.chat.completions.create(
549 - model=model_name,
550 - messages=messages,
551 - max_tokens=10,
552 - temperature=0.0,
553 - stream=True,
554 - stream_options={"include_usage": True})
555 -
556 - async for chunk in stream:
557 - if chunk.choices[0].finish_reason is None:
558 - assert chunk.usage is None
559 - else:
560 - assert chunk.usage is None
561 - final_chunk = await stream.__anext__()
562 - assert final_chunk.usage is not None
563 - assert final_chunk.usage.prompt_tokens > 0
564 - assert final_chunk.usage.completion_tokens > 0
565 - assert final_chunk.usage.total_tokens == (
566 - final_chunk.usage.prompt_tokens +
567 - final_chunk.usage.completion_tokens)
568 - assert final_chunk.choices == []
569 -
570 - # Test stream=False, stream_options={"include_usage": None}
571 - with pytest.raises(BadRequestError):
572 - await client.chat.completions.create(
573 - model=model_name,
574 - messages=messages,
575 - max_tokens=10,
576 - temperature=0.0,
577 - stream=False,
578 - stream_options={"include_usage": None})
579 -
580 - # Test stream=False, stream_options={"include_usage": True}
581 - with pytest.raises(BadRequestError):
582 - await client.chat.completions.create(
583 - model=model_name,
584 - messages=messages,
585 - max_tokens=10,
586 - temperature=0.0,
587 - stream=False,
588 - stream_options={"include_usage": True})
589 -
590 -
303 @pytest.mark.asyncio
304 @pytest.mark.parametrize(
305 "model_name",
@@ -773,53 +485,6 @@ async def test_guided_json_completion(client: openai.AsyncOpenAI,
485 jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
486
487
776 -@pytest.mark.asyncio
777 -@pytest.mark.parametrize("guided_decoding_backend",
778 - ["outlines", "lm-format-enforcer"])
779 -async def test_guided_json_chat(client: openai.AsyncOpenAI,
780 - guided_decoding_backend: str):
781 - messages = [{
782 - "role": "system",
783 - "content": "you are a helpful assistant"
784 - }, {
785 - "role":
786 - "user",
787 - "content":
788 - f"Give an example JSON for an employee profile that "
789 - f"fits this schema: {TEST_SCHEMA}"
790 - }]
791 - chat_completion = await client.chat.completions.create(
792 - model=MODEL_NAME,
793 - messages=messages,
794 - max_tokens=1000,
795 - extra_body=dict(guided_json=TEST_SCHEMA,
796 - guided_decoding_backend=guided_decoding_backend))
797 - message = chat_completion.choices[0].message
798 - assert message.content is not None
799 - json1 = json.loads(message.content)
800 - jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
801 -
802 - messages.append({"role": "assistant", "content": message.content})
803 - messages.append({
804 - "role":
805 - "user",
806 - "content":
807 - "Give me another one with a different name and age"
808 - })
809 - chat_completion = await client.chat.completions.create(
810 - model=MODEL_NAME,
811 - messages=messages,
812 - max_tokens=1000,
813 - extra_body=dict(guided_json=TEST_SCHEMA,
814 - guided_decoding_backend=guided_decoding_backend))
815 - message = chat_completion.choices[0].message
816 - assert message.content is not None
817 - json2 = json.loads(message.content)
818 - jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
819 - assert json1["name"] != json2["name"]
820 - assert json1["age"] != json2["age"]
821 -
822 -
488 @pytest.mark.asyncio
489 @pytest.mark.parametrize("guided_decoding_backend",
490 ["outlines", "lm-format-enforcer"])
@@ -840,44 +505,6 @@ async def test_guided_regex_completion(client: openai.AsyncOpenAI,
505 assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
506
507
843 -@pytest.mark.asyncio
844 -@pytest.mark.parametrize("guided_decoding_backend",
845 - ["outlines", "lm-format-enforcer"])
846 -async def test_guided_regex_chat(client: openai.AsyncOpenAI,
847 - guided_decoding_backend: str):
848 - messages = [{
849 - "role": "system",
850 - "content": "you are a helpful assistant"
851 - }, {
852 - "role":
853 - "user",
854 - "content":
855 - f"Give an example IP address with this regex: {TEST_REGEX}"
856 - }]
857 - chat_completion = await client.chat.completions.create(
858 - model=MODEL_NAME,
859 - messages=messages,
860 - max_tokens=20,
861 - extra_body=dict(guided_regex=TEST_REGEX,
862 - guided_decoding_backend=guided_decoding_backend))
863 - ip1 = chat_completion.choices[0].message.content
864 - assert ip1 is not None
865 - assert re.fullmatch(TEST_REGEX, ip1) is not None
866 -
867 - messages.append({"role": "assistant", "content": ip1})
868 - messages.append({"role": "user", "content": "Give me a different one"})
869 - chat_completion = await client.chat.completions.create(
870 - model=MODEL_NAME,
871 - messages=messages,
872 - max_tokens=20,
873 - extra_body=dict(guided_regex=TEST_REGEX,
874 - guided_decoding_backend=guided_decoding_backend))
875 - ip2 = chat_completion.choices[0].message.content
876 - assert ip2 is not None
877 - assert re.fullmatch(TEST_REGEX, ip2) is not None
878 - assert ip1 != ip2
879 -
880 -
508 @pytest.mark.asyncio
509 @pytest.mark.parametrize("guided_decoding_backend",
510 ["outlines", "lm-format-enforcer"])
@@ -898,385 +525,6 @@ async def test_guided_choice_completion(client: openai.AsyncOpenAI,
525 assert completion.choices[i].text in TEST_CHOICE
526
527
901 -@pytest.mark.asyncio
902 -@pytest.mark.parametrize("guided_decoding_backend",
903 - ["outlines", "lm-format-enforcer"])
904 -async def test_guided_choice_chat(client: openai.AsyncOpenAI,
905 - guided_decoding_backend: str):
906 - messages = [{
907 - "role": "system",
908 - "content": "you are a helpful assistant"
909 - }, {
910 - "role":
911 - "user",
912 - "content":
913 - "The best language for type-safe systems programming is "
914 - }]
915 - chat_completion = await client.chat.completions.create(
916 - model=MODEL_NAME,
917 - messages=messages,
918 - max_tokens=10,
919 - extra_body=dict(guided_choice=TEST_CHOICE,
920 - guided_decoding_backend=guided_decoding_backend))
921 - choice1 = chat_completion.choices[0].message.content
922 - assert choice1 in TEST_CHOICE
923 -
924 - messages.append({"role": "assistant", "content": choice1})
925 - messages.append({
926 - "role": "user",
927 - "content": "I disagree, pick another one"
928 - })
929 - chat_completion = await client.chat.completions.create(
930 - model=MODEL_NAME,
931 - messages=messages,
932 - max_tokens=10,
933 - extra_body=dict(guided_choice=TEST_CHOICE,
934 - guided_decoding_backend=guided_decoding_backend))
935 - choice2 = chat_completion.choices[0].message.content
936 - assert choice2 in TEST_CHOICE
937 - assert choice1 != choice2
938 -
939 -
940 -@pytest.mark.asyncio
941 -@pytest.mark.parametrize("guided_decoding_backend",
942 - ["outlines", "lm-format-enforcer"])
943 -async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
944 - guided_decoding_backend: str):
945 - with pytest.raises(openai.BadRequestError):
946 - _ = await client.completions.create(
947 - model=MODEL_NAME,
948 - prompt="Give an example JSON that fits this schema: 42",
949 - extra_body=dict(guided_json=42,
950 - guided_decoding_backend=guided_decoding_backend))
951 -
952 - messages = [{
953 - "role": "system",
954 - "content": "you are a helpful assistant"
955 - }, {
956 - "role":
957 - "user",
958 - "content":
959 - "The best language for type-safe systems programming is "
960 - }]
961 - with pytest.raises(openai.BadRequestError):
962 - _ = await client.chat.completions.create(model=MODEL_NAME,
963 - messages=messages,
964 - extra_body=dict(guided_regex={
965 - 1: "Python",
966 - 2: "C++"
967 - }))
968 -
969 - with pytest.raises(openai.BadRequestError):
970 - _ = await client.completions.create(
971 - model=MODEL_NAME,
972 - prompt="Give an example string that fits this regex",
973 - extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
974 -
975 -
976 -@pytest.mark.asyncio
977 -@pytest.mark.parametrize("guided_decoding_backend",
978 - ["outlines", "lm-format-enforcer"])
979 -async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
980 - guided_decoding_backend: str):
981 - messages = [{
982 - "role": "system",
983 - "content": "you are a helpful assistant"
984 - }, {
985 - "role":
986 - "user",
987 - "content":
988 - "The best language for type-safe systems programming is "
989 - }]
990 - chat_completion = await client.chat.completions.create(
991 - model=MODEL_NAME,
992 - messages=messages,
993 - max_tokens=10,
994 - logprobs=True,
995 - top_logprobs=5,
996 - extra_body=dict(guided_choice=TEST_CHOICE,
997 - guided_decoding_backend=guided_decoding_backend))
998 -
999 - assert chat_completion.choices[0].logprobs is not None
1000 - assert chat_completion.choices[0].logprobs.content is not None
1001 - top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
1002 -
1003 - # -9999.0 is the minimum logprob returned by OpenAI
1004 - for item in top_logprobs:
1005 - assert item.logprob >= -9999.0, f"Failed (top_logprobs={top_logprobs})"
1006 -
1007 -
1008 -@pytest.mark.asyncio
1009 -@pytest.mark.parametrize("guided_decoding_backend",
1010 - ["outlines", "lm-format-enforcer"])
1011 -async def test_named_tool_use(client: openai.AsyncOpenAI,
1012 - guided_decoding_backend: str):
1013 - messages = [{
1014 - "role": "system",
1015 - "content": "you are a helpful assistant"
1016 - }, {
1017 - "role":
1018 - "user",
1019 - "content":
1020 - f"Give an example JSON for an employee profile that "
1021 - f"fits this schema: {TEST_SCHEMA}"
1022 - }]
1023 -
1024 - # non-streaming
1025 -
1026 - chat_completion = await client.chat.completions.create(
1027 - model=MODEL_NAME,
1028 - messages=messages,
1029 - max_tokens=1000,
1030 - tools=[{
1031 - "type": "function",
1032 - "function": {
1033 - "name": "dummy_function_name",
1034 - "description": "This is a dummy function",
1035 - "parameters": TEST_SCHEMA
1036 - }
1037 - }],
1038 - tool_choice={
1039 - "type": "function",
1040 - "function": {
1041 - "name": "dummy_function_name"
1042 - }
1043 - })
1044 - message = chat_completion.choices[0].message
1045 - assert len(message.content) == 0
1046 - json_string = message.tool_calls[0].function.arguments
1047 - json1 = json.loads(json_string)
1048 - jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
1049 -
1050 - messages.append({"role": "assistant", "content": json_string})
1051 - messages.append({
1052 - "role":
1053 - "user",
1054 - "content":
1055 - "Give me another one with a different name and age"
1056 - })
1057 -
1058 - # streaming
1059 -
1060 - stream = await client.chat.completions.create(
1061 - model=MODEL_NAME,
1062 - messages=messages,
1063 - max_tokens=1000,
1064 - tools=[{
1065 - "type": "function",
1066 - "function": {
1067 - "name": "dummy_function_name",
1068 - "description": "This is a dummy function",
1069 - "parameters": TEST_SCHEMA
1070 - }
1071 - }],
1072 - tool_choice={
1073 - "type": "function",
1074 - "function": {
1075 - "name": "dummy_function_name"
1076 - }
1077 - },
1078 - stream=True)
1079 -
1080 - output = []
1081 - finish_reason_count = 0
1082 - async for chunk in stream:
1083 - delta = chunk.choices[0].delta
1084 - if delta.role:
1085 - assert delta.role == "assistant"
1086 - assert delta.content is None or len(delta.content) == 0
1087 - if delta.tool_calls:
1088 - output.append(delta.tool_calls[0].function.arguments)
1089 - if chunk.choices[0].finish_reason is not None:
1090 - finish_reason_count += 1
1091 - # finish reason should only return in last block
1092 - assert finish_reason_count == 1
1093 - json2 = json.loads("".join(output))
1094 - jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
1095 - assert json1["name"] != json2["name"]
1096 - assert json1["age"] != json2["age"]
1097 -
1098 -
1099 -@pytest.mark.asyncio
1100 -@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1101 -async def test_required_tool_use_not_yet_supported(
1102 - client: openai.AsyncOpenAI, guided_decoding_backend: str):
1103 - messages = [{
1104 - "role": "system",
1105 - "content": "you are a helpful assistant"
1106 - }, {
1107 - "role":
1108 - "user",
1109 - "content":
1110 - f"Give an example JSON for an employee profile that "
1111 - f"fits this schema: {TEST_SCHEMA}"
1112 - }]
1113 -
1114 - with pytest.raises(openai.BadRequestError):
1115 - await client.chat.completions.create(
1116 - model=MODEL_NAME,
1117 - messages=messages,
1118 - max_tokens=1000,
1119 - tools=[{
1120 - "type": "function",
1121 - "function": {
1122 - "name": "dummy_function_name",
1123 - "description": "This is a dummy function",
1124 - "parameters": TEST_SCHEMA
1125 - }
1126 - }],
1127 - tool_choice="required")
1128 -
1129 - with pytest.raises(openai.BadRequestError):
1130 - await client.chat.completions.create(
1131 - model=MODEL_NAME,
1132 - messages=messages,
1133 - max_tokens=1000,
1134 - tools=[{
1135 - "type": "function",
1136 - "function": {
1137 - "name": "dummy_function_name",
1138 - "description": "This is a dummy function",
1139 - "parameters": TEST_SCHEMA
1140 - }
1141 - }],
1142 - tool_choice="auto")
1143 -
1144 -
1145 -@pytest.mark.asyncio
1146 -@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1147 -async def test_inconsistent_tool_choice_and_tools(
1148 - client: openai.AsyncOpenAI, guided_decoding_backend: str):
1149 - messages = [{
1150 - "role": "system",
1151 - "content": "you are a helpful assistant"
1152 - }, {
1153 - "role":
1154 - "user",
1155 - "content":
1156 - f"Give an example JSON for an employee profile that "
1157 - f"fits this schema: {TEST_SCHEMA}"
1158 - }]
1159 -
1160 - with pytest.raises(openai.BadRequestError):
1161 - await client.chat.completions.create(model=MODEL_NAME,
1162 - messages=messages,
1163 - max_tokens=1000,
1164 - tool_choice={
1165 - "type": "function",
1166 - "function": {
1167 - "name":
1168 - "dummy_function_name"
1169 - }
1170 - })
1171 -
1172 - with pytest.raises(openai.BadRequestError):
1173 - await client.chat.completions.create(
1174 - model=MODEL_NAME,
1175 - messages=messages,
1176 - max_tokens=1000,
1177 - tools=[{
1178 - "type": "function",
1179 - "function": {
1180 - "name": "dummy_function_name",
1181 - "description": "This is a dummy function",
1182 - "parameters": TEST_SCHEMA
1183 - }
1184 - }],
1185 - tool_choice={
1186 - "type": "function",
1187 - "function": {
1188 - "name": "nondefined_function_name"
1189 - }
1190 - })
1191 -
1192 -
1193 -@pytest.mark.asyncio
1194 -async def test_response_format_json_object(client: openai.AsyncOpenAI):
1195 - for _ in range(2):
1196 - resp = await client.chat.completions.create(
1197 - model=MODEL_NAME,
1198 - messages=[{
1199 - "role":
1200 - "user",
1201 - "content": ('what is 1+1? please respond with a JSON object, '
1202 - 'the format is {"result": 2}')
1203 - }],
1204 - response_format={"type": "json_object"})
1205 -
1206 - content = resp.choices[0].message.content
1207 - assert content is not None
1208 -
1209 - loaded = json.loads(content)
1210 - assert loaded == {"result": 2}, loaded
1211 -
1212 -
1213 -@pytest.mark.asyncio
1214 -async def test_extra_fields(client: openai.AsyncOpenAI):
1215 - with pytest.raises(BadRequestError) as exc_info:
1216 - await client.chat.completions.create(
1217 - model=MODEL_NAME,
1218 - messages=[{
1219 - "role": "system",
1220 - "content": "You are a helpful assistant.",
1221 - "extra_field": "0",
1222 - }], # type: ignore
1223 - temperature=0,
1224 - seed=0)
1225 -
1226 - assert "extra_forbidden" in exc_info.value.message
1227 -
1228 -
1229 -@pytest.mark.asyncio
1230 -async def test_complex_message_content(client: openai.AsyncOpenAI):
1231 - resp = await client.chat.completions.create(
1232 - model=MODEL_NAME,
1233 - messages=[{
1234 - "role":
1235 - "user",
1236 - "content": [{
1237 - "type":
1238 - "text",
1239 - "text":
1240 - "what is 1+1? please provide the result without any other text."
1241 - }]
1242 - }],
1243 - temperature=0,
1244 - seed=0)
1245 - content = resp.choices[0].message.content
1246 - assert content == "2"
1247 -
1248 -
1249 -@pytest.mark.asyncio
1250 -async def test_custom_role(client: openai.AsyncOpenAI):
1251 - # Not sure how the model handles custom roles so we just check that
1252 - # both string and complex message content are handled in the same way
1253 -
1254 - resp1 = await client.chat.completions.create(
1255 - model=MODEL_NAME,
1256 - messages=[{
1257 - "role": "my-custom-role",
1258 - "content": "what is 1+1?",
1259 - }], # type: ignore
1260 - temperature=0,
1261 - seed=0)
1262 -
1263 - resp2 = await client.chat.completions.create(
1264 - model=MODEL_NAME,
1265 - messages=[{
1266 - "role": "my-custom-role",
1267 - "content": [{
1268 - "type": "text",
1269 - "text": "what is 1+1?"
1270 - }]
1271 - }], # type: ignore
1272 - temperature=0,
1273 - seed=0)
1274 -
1275 - content1 = resp1.choices[0].message.content
1276 - content2 = resp2.choices[0].message.content
1277 - assert content1 == content2
1278 -
1279 -
528 @pytest.mark.asyncio
529 async def test_guided_grammar(client: openai.AsyncOpenAI):
530 simple_sql_grammar = """
@@ -1348,23 +596,22 @@ async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
596
597
598 @pytest.mark.asyncio
1351 -async def test_long_seed(client: openai.AsyncOpenAI):
1352 - for seed in [
1353 - torch.iinfo(torch.long).min - 1,
1354 - torch.iinfo(torch.long).max + 1
1355 - ]:
1356 - with pytest.raises(BadRequestError) as exc_info:
1357 - await client.chat.completions.create(
1358 - model=MODEL_NAME,
1359 - messages=[{
1360 - "role": "system",
1361 - "content": "You are a helpful assistant.",
1362 - }],
1363 - temperature=0,
1364 - seed=seed)
1365 -
1366 - assert ("greater_than_equal" in exc_info.value.message
1367 - or "less_than_equal" in exc_info.value.message)
599 +@pytest.mark.parametrize("guided_decoding_backend",
600 + ["outlines", "lm-format-enforcer"])
601 +async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
602 + guided_decoding_backend: str):
603 + with pytest.raises(openai.BadRequestError):
604 + _ = await client.completions.create(
605 + model=MODEL_NAME,
606 + prompt="Give an example JSON that fits this schema: 42",
607 + extra_body=dict(guided_json=42,
608 + guided_decoding_backend=guided_decoding_backend))
609 +
610 + with pytest.raises(openai.BadRequestError):
611 + _ = await client.completions.create(
612 + model=MODEL_NAME,
613 + prompt="Give an example string that fits this regex",
614 + extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
615
616
617 @pytest.mark.asyncio
@@ -1372,7 +619,7 @@ async def test_long_seed(client: openai.AsyncOpenAI):
619 "model_name",
620 [MODEL_NAME],
621 )
1375 -async def test_tokenize(server, client: openai.AsyncOpenAI, model_name: str):
622 +async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
623 base_url = str(client.base_url)[:-3]
624 tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
625
@@ -1399,7 +646,7 @@ async def test_tokenize(server, client: openai.AsyncOpenAI, model_name: str):
646 "model_name",
647 [MODEL_NAME],
648 )
1402 -async def test_detokenize(server, client: openai.AsyncOpenAI, model_name: str):
649 +async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
650 base_url = str(client.base_url)[:-3]
651 tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
652
tests/entrypoints/test_openai_vision.py
+2 -2
@@ -1,5 +1,5 @@
1 from pathlib import Path
2 -from typing import Dict
2 +from typing import Dict, List
3
4 import openai
5 import pytest
@@ -216,7 +216,7 @@ async def test_chat_streaming_image(client: openai.AsyncOpenAI,
216 temperature=0.0,
217 stream=True,
218 )
219 - chunks = []
219 + chunks: List[str] = []
220 finish_reason_count = 0
221 async for chunk in stream:
222 delta = chunk.choices[0].delta