A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[ci][frontend] deduplicate tests (#7101)
youkaichao committed
Aug 5, 2024 at 15:59 UTC
dfb1a15dcb4c24bf7ff0ba7ddfc5d623ad519d7d
2 files changed
+6
-723
tests/entrypoints/openai/test_completion.py
+6
-8
@@ -87,15 +87,13 @@ def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files,
87
]
88
89
90
-@pytest.fixture(scope="module")
91
-def server(default_server_args):
90
+@pytest.fixture(scope="module",
91
+ params=["", "--disable-frontend-multiprocessing"])
92
+def client(default_server_args, request):
93
+ if request.param:
94
+ default_server_args.append(request.param)
95
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
93
- yield remote_server
94
-
95
-
96
-@pytest.fixture(scope="module")
97
-def client(server):
98
- return server.get_async_client()
96
+ yield remote_server.get_async_client()
97
98
99
@pytest.mark.asyncio
tests/entrypoints/openai/test_disable_mp.py
deleted
-715
@@ -1,715 +0,0 @@
1
-"""
2
-Repeat of tests in test_completion.py with the non-mp backend.
3
-"""
4
-
5
-# imports for guided decoding tests
6
-import json
7
-import re
8
-import shutil
9
-from tempfile import TemporaryDirectory
10
-from typing import List
11
-
12
-import jsonschema
13
-import openai # use the official client for correctness check
14
-import pytest
15
-# downloading lora to test lora requests
16
-from huggingface_hub import snapshot_download
17
-from openai import BadRequestError
18
-from transformers import AutoTokenizer
19
-
20
-from vllm.transformers_utils.tokenizer import get_tokenizer
21
-
22
-from ...utils import RemoteOpenAIServer
23
-
24
-# any model with a chat template should work here
25
-MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
26
-# technically these adapters use a different base model,
27
-# but we're not testing generation quality here
28
-LORA_NAME = "typeof/zephyr-7b-beta-lora"
29
-PA_NAME = "swapnilbp/llama_tweet_ptune"
30
-# if PA_NAME changes, PA_NUM_VIRTUAL_TOKENS might also
31
-# need to change to match the prompt adapter
32
-PA_NUM_VIRTUAL_TOKENS = 8
33
-
34
-
35
-@pytest.fixture(scope="module")
36
-def zephyr_lora_files():
37
- return snapshot_download(repo_id=LORA_NAME)
38
-
39
-
40
-@pytest.fixture(scope="module")
41
-def zephyr_lora_added_tokens_files(zephyr_lora_files):
42
- tmp_dir = TemporaryDirectory()
43
- tmp_model_dir = f"{tmp_dir.name}/zephyr"
44
- shutil.copytree(zephyr_lora_files, tmp_model_dir)
45
- tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
46
- # Copy tokenizer to adapter and add some unique tokens
47
- # 32000, 32001, 32002
48
- added = tokenizer.add_tokens(["vllm1", "vllm2", "vllm3"],
49
- special_tokens=True)
50
- assert added == 3
51
- tokenizer.save_pretrained(tmp_model_dir)
52
- yield tmp_model_dir
53
- tmp_dir.cleanup()
54
-
55
-
56
-@pytest.fixture(scope="module")
57
-def zephyr_pa_files():
58
- return snapshot_download(repo_id=PA_NAME)
59
-
60
-
61
-@pytest.fixture(scope="module")
62
-def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files,
63
- zephyr_pa_files):
64
- return [
65
- # use half precision for speed and memory savings in CI environment
66
- "--dtype",
67
- "bfloat16",
68
- "--max-model-len",
69
- "8192",
70
- "--max-num-seqs",
71
- "128",
72
- "--enforce-eager",
73
- # lora config
74
- "--enable-lora",
75
- "--lora-modules",
76
- f"zephyr-lora={zephyr_lora_files}",
77
- f"zephyr-lora2={zephyr_lora_added_tokens_files}",
78
- "--max-lora-rank",
79
- "64",
80
- "--max-cpu-loras",
81
- "2",
82
- # pa config
83
- "--enable-prompt-adapter",
84
- "--prompt-adapters",
85
- f"zephyr-pa={zephyr_pa_files}",
86
- f"zephyr-pa2={zephyr_pa_files}",
87
- "--max-prompt-adapters",
88
- "2",
89
- "--max-prompt-adapter-token",
90
- "128",
91
- "--disable-frontend-multiprocessing"
92
- ]
93
-
94
-
95
-@pytest.fixture(scope="module")
96
-def server(default_server_args):
97
- with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
98
- yield remote_server
99
-
100
-
101
-@pytest.fixture(scope="module")
102
-def client(server):
103
- return server.get_async_client()
104
-
105
-
106
-@pytest.mark.asyncio
107
-@pytest.mark.parametrize(
108
- # first test base model, then test loras, then test prompt adapters
109
- "model_name,num_virtual_tokens",
110
- [(MODEL_NAME, 0), ("zephyr-lora", 0), ("zephyr-lora2", 0),
111
- ("zephyr-pa", PA_NUM_VIRTUAL_TOKENS),
112
- ("zephyr-pa2", PA_NUM_VIRTUAL_TOKENS)],
113
-)
114
-async def test_single_completion(client: openai.AsyncOpenAI, model_name: str,
115
- num_virtual_tokens: int):
116
- completion = await client.completions.create(model=model_name,
117
- prompt="Hello, my name is",
118
- max_tokens=5,
119
- temperature=0.0)
120
-
121
- assert completion.id is not None
122
- assert completion.choices is not None and len(completion.choices) == 1
123
-
124
- choice = completion.choices[0]
125
- assert len(choice.text) >= 5
126
- assert choice.finish_reason == "length"
127
- assert completion.usage == openai.types.CompletionUsage(
128
- completion_tokens=5,
129
- prompt_tokens=6 + num_virtual_tokens,
130
- total_tokens=11 + num_virtual_tokens)
131
-
132
- # test using token IDs
133
- completion = await client.completions.create(
134
- model=model_name,
135
- prompt=[0, 0, 0, 0, 0],
136
- max_tokens=5,
137
- temperature=0.0,
138
- )
139
- assert len(completion.choices[0].text) >= 1
140
-
141
-
142
-@pytest.mark.asyncio
143
-async def test_added_lora_tokens(client: openai.AsyncOpenAI):
144
- # test using token IDs
145
- completion = await client.completions.create(
146
- model="zephyr-lora2",
147
- prompt=[0, 0, 32000, 32001, 32002],
148
- echo=True,
149
- max_tokens=5,
150
- temperature=0.0,
151
- )
152
- # Added tokens should appear in tokenized prompt
153
- assert completion.choices[0].text.startswith("<unk><unk>vllm1vllm2vllm3")
154
-
155
-
156
-@pytest.mark.asyncio
157
-async def test_added_lora_tokens_base_model(client: openai.AsyncOpenAI):
158
- # test using token IDs
159
- completion = await client.completions.create(
160
- model=MODEL_NAME,
161
- prompt=[0, 0, 32000, 32001, 32002],
162
- echo=True,
163
- max_tokens=5,
164
- temperature=0.0,
165
- )
166
- # Added tokens should not appear in tokenized prompt
167
- assert "vllm" not in completion.choices[0].text
168
-
169
-
170
-@pytest.mark.asyncio
171
-@pytest.mark.parametrize(
172
- # first test base model, then test loras, then test prompt adapters
173
- "model_name",
174
- [MODEL_NAME, "zephyr-lora", "zephyr-lora2", "zephyr-pa", "zephyr-pa2"],
175
-)
176
-async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
177
- # test using token IDs
178
- completion = await client.completions.create(
179
- model=model_name,
180
- prompt=[0, 0, 0, 0, 0],
181
- max_tokens=5,
182
- temperature=0.0,
183
- logprobs=None,
184
- )
185
- choice = completion.choices[0]
186
- assert choice.logprobs is None
187
-
188
-
189
-@pytest.mark.asyncio
190
-@pytest.mark.parametrize(
191
- # just test 1 lora and 1 pa hereafter
192
- "model_name",
193
- [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
194
-)
195
-async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
196
- # test using token IDs
197
- completion = await client.completions.create(
198
- model=model_name,
199
- prompt=[0, 0, 0, 0, 0],
200
- max_tokens=5,
201
- temperature=0.0,
202
- logprobs=0,
203
- )
204
- choice = completion.choices[0]
205
- assert choice.logprobs is not None
206
- assert choice.logprobs.token_logprobs is not None
207
- assert choice.logprobs.top_logprobs is not None
208
- assert len(choice.logprobs.top_logprobs[0]) == 1
209
-
210
-
211
-@pytest.mark.asyncio
212
-@pytest.mark.parametrize(
213
- "model_name",
214
- [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
215
-)
216
-async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
217
- # test using token IDs
218
- completion = await client.completions.create(
219
- model=model_name,
220
- prompt=[0, 0, 0, 0, 0],
221
- max_tokens=5,
222
- temperature=0.0,
223
- logprobs=5,
224
- )
225
- choice = completion.choices[0]
226
- assert choice.logprobs is not None
227
- assert choice.logprobs.token_logprobs is not None
228
- assert choice.logprobs.top_logprobs is not None
229
- assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
230
-
231
-
232
-@pytest.mark.asyncio
233
-@pytest.mark.parametrize(
234
- "model_name",
235
- [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
236
-)
237
-async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
238
- model_name: str):
239
-
240
- with pytest.raises(
241
- (openai.BadRequestError, openai.APIError)): # test using token IDs
242
- await client.completions.create(
243
- model=model_name,
244
- prompt=[0, 0, 0, 0, 0],
245
- max_tokens=5,
246
- temperature=0.0,
247
- # vLLM has higher default max_logprobs (20 instead of 5) to support
248
- # both Completion API and Chat Completion API
249
- logprobs=21,
250
- )
251
- ...
252
- with pytest.raises(
253
- (openai.BadRequestError, openai.APIError)): # test using token IDs
254
- stream = await client.completions.create(
255
- model=model_name,
256
- prompt=[0, 0, 0, 0, 0],
257
- max_tokens=5,
258
- temperature=0.0,
259
- # vLLM has higher default max_logprobs (20 instead of 5) to support
260
- # both Completion API and Chat Completion API
261
- logprobs=30,
262
- stream=True,
263
- )
264
- async for chunk in stream:
265
- ...
266
-
267
- # the server should still work afterwards
268
- completion = await client.completions.create(
269
- model=model_name,
270
- prompt=[0, 0, 0, 0, 0],
271
- max_tokens=5,
272
- temperature=0.0,
273
- )
274
- assert len(completion.choices[0].text) >= 0
275
-
276
-
277
-@pytest.mark.asyncio
278
-@pytest.mark.parametrize(
279
- "model_name",
280
- [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
281
-)
282
-async def test_completion_streaming(client: openai.AsyncOpenAI,
283
- model_name: str):
284
- prompt = "What is an LLM?"
285
-
286
- single_completion = await client.completions.create(
287
- model=model_name,
288
- prompt=prompt,
289
- max_tokens=5,
290
- temperature=0.0,
291
- )
292
- single_output = single_completion.choices[0].text
293
- stream = await client.completions.create(model=model_name,
294
- prompt=prompt,
295
- max_tokens=5,
296
- temperature=0.0,
297
- stream=True)
298
- chunks: List[str] = []
299
- finish_reason_count = 0
300
- async for chunk in stream:
301
- chunks.append(chunk.choices[0].text)
302
- if chunk.choices[0].finish_reason is not None:
303
- finish_reason_count += 1
304
- # finish reason should only return in last block
305
- assert finish_reason_count == 1
306
- assert chunk.choices[0].finish_reason == "length"
307
- assert chunk.choices[0].text
308
- assert "".join(chunks) == single_output
309
-
310
-
311
-@pytest.mark.asyncio
312
-@pytest.mark.parametrize(
313
- "model_name",
314
- [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
315
-)
316
-async def test_completion_stream_options(client: openai.AsyncOpenAI,
317
- model_name: str):
318
- prompt = "What is the capital of France?"
319
-
320
- # Test stream=True, stream_options=
321
- # {"include_usage": False, "continuous_usage_stats": False}
322
- stream = await client.completions.create(model=model_name,
323
- prompt=prompt,
324
- max_tokens=5,
325
- temperature=0.0,
326
- stream=True,
327
- stream_options={
328
- "include_usage": False,
329
- "continuous_usage_stats":
330
- False,
331
- })
332
-
333
- async for chunk in stream:
334
- assert chunk.usage is None
335
-
336
- # Test stream=True, stream_options=
337
- # {"include_usage": False, "continuous_usage_stats": True}
338
- stream = await client.completions.create(model=model_name,
339
- prompt=prompt,
340
- max_tokens=5,
341
- temperature=0.0,
342
- stream=True,
343
- stream_options={
344
- "include_usage": False,
345
- "continuous_usage_stats":
346
- True,
347
- })
348
- async for chunk in stream:
349
- assert chunk.usage is None
350
-
351
- # Test stream=True, stream_options=
352
- # {"include_usage": True, "continuous_usage_stats": False}
353
- stream = await client.completions.create(model=model_name,
354
- prompt=prompt,
355
- max_tokens=5,
356
- temperature=0.0,
357
- stream=True,
358
- stream_options={
359
- "include_usage": True,
360
- "continuous_usage_stats":
361
- False,
362
- })
363
- async for chunk in stream:
364
- if chunk.choices[0].finish_reason is None:
365
- assert chunk.usage is None
366
- else:
367
- assert chunk.usage is None
368
- final_chunk = await stream.__anext__()
369
- assert final_chunk.usage is not None
370
- assert final_chunk.usage.prompt_tokens > 0
371
- assert final_chunk.usage.completion_tokens > 0
372
- assert final_chunk.usage.total_tokens == (
373
- final_chunk.usage.prompt_tokens +
374
- final_chunk.usage.completion_tokens)
375
- assert final_chunk.choices == []
376
-
377
- # Test stream=True, stream_options=
378
- # {"include_usage": True, "continuous_usage_stats": True}
379
- stream = await client.completions.create(model=model_name,
380
- prompt=prompt,
381
- max_tokens=5,
382
- temperature=0.0,
383
- stream=True,
384
- stream_options={
385
- "include_usage": True,
386
- "continuous_usage_stats":
387
- True,
388
- })
389
- async for chunk in stream:
390
- assert chunk.usage is not None
391
- assert chunk.usage.prompt_tokens > 0
392
- assert chunk.usage.completion_tokens > 0
393
- assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
394
- chunk.usage.completion_tokens)
395
- if chunk.choices[0].finish_reason is not None:
396
- final_chunk = await stream.__anext__()
397
- assert final_chunk.usage is not None
398
- assert final_chunk.usage.prompt_tokens > 0
399
- assert final_chunk.usage.completion_tokens > 0
400
- assert final_chunk.usage.total_tokens == (
401
- final_chunk.usage.prompt_tokens +
402
- final_chunk.usage.completion_tokens)
403
- assert final_chunk.choices == []
404
-
405
- # Test stream=False, stream_options=
406
- # {"include_usage": None}
407
- with pytest.raises(BadRequestError):
408
- await client.completions.create(model=model_name,
409
- prompt=prompt,
410
- max_tokens=5,
411
- temperature=0.0,
412
- stream=False,
413
- stream_options={"include_usage": None})
414
-
415
- # Test stream=False, stream_options=
416
- # {"include_usage": True}
417
- with pytest.raises(BadRequestError):
418
- await client.completions.create(model=model_name,
419
- prompt=prompt,
420
- max_tokens=5,
421
- temperature=0.0,
422
- stream=False,
423
- stream_options={"include_usage": True})
424
-
425
- # Test stream=False, stream_options=
426
- # {"continuous_usage_stats": None}
427
- with pytest.raises(BadRequestError):
428
- await client.completions.create(
429
- model=model_name,
430
- prompt=prompt,
431
- max_tokens=5,
432
- temperature=0.0,
433
- stream=False,
434
- stream_options={"continuous_usage_stats": None})
435
-
436
- # Test stream=False, stream_options=
437
- # {"continuous_usage_stats": True}
438
- with pytest.raises(BadRequestError):
439
- await client.completions.create(
440
- model=model_name,
441
- prompt=prompt,
442
- max_tokens=5,
443
- temperature=0.0,
444
- stream=False,
445
- stream_options={"continuous_usage_stats": True})
446
-
447
-
448
-@pytest.mark.asyncio
449
-@pytest.mark.parametrize(
450
- "model_name",
451
- [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
452
-)
453
-async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
454
- # test both text and token IDs
455
- for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
456
- # test simple list
457
- batch = await client.completions.create(
458
- model=model_name,
459
- prompt=prompts,
460
- max_tokens=5,
461
- temperature=0.0,
462
- )
463
- assert len(batch.choices) == 2
464
- assert batch.choices[0].text == batch.choices[1].text
465
-
466
- # test n = 2
467
- batch = await client.completions.create(
468
- model=model_name,
469
- prompt=prompts,
470
- n=2,
471
- max_tokens=5,
472
- temperature=0.0,
473
- extra_body=dict(
474
- # NOTE: this has to be true for n > 1 in vLLM, but not necessary
475
- # for official client.
476
- use_beam_search=True),
477
- )
478
- assert len(batch.choices) == 4
479
- assert batch.choices[0].text != batch.choices[
480
- 1].text, "beam search should be different"
481
- assert batch.choices[0].text == batch.choices[
482
- 2].text, "two copies of the same prompt should be the same"
483
- assert batch.choices[1].text == batch.choices[
484
- 3].text, "two copies of the same prompt should be the same"
485
-
486
- # test streaming
487
- batch = await client.completions.create(
488
- model=model_name,
489
- prompt=prompts,
490
- max_tokens=5,
491
- temperature=0.0,
492
- stream=True,
493
- )
494
- texts = [""] * 2
495
- async for chunk in batch:
496
- assert len(chunk.choices) == 1
497
- choice = chunk.choices[0]
498
- texts[choice.index] += choice.text
499
- assert texts[0] == texts[1]
500
-
501
-
502
-@pytest.mark.asyncio
503
-async def test_logits_bias(client: openai.AsyncOpenAI):
504
- prompt = "Hello, my name is"
505
- max_tokens = 5
506
- tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
507
-
508
- # Test exclusive selection
509
- token_id = 1000
510
- completion = await client.completions.create(
511
- model=MODEL_NAME,
512
- prompt=prompt,
513
- max_tokens=max_tokens,
514
- temperature=0.0,
515
- logit_bias={str(token_id): 100},
516
- seed=42,
517
- )
518
- assert len(completion.choices[0].text) >= 5
519
- response_tokens = tokenizer(completion.choices[0].text,
520
- add_special_tokens=False)["input_ids"]
521
- expected_tokens = tokenizer(tokenizer.decode([token_id] * 5),
522
- add_special_tokens=False)["input_ids"]
523
- assert all([
524
- response == expected
525
- for response, expected in zip(response_tokens, expected_tokens)
526
- ])
527
-
528
- # Test ban
529
- completion = await client.completions.create(
530
- model=MODEL_NAME,
531
- prompt=prompt,
532
- max_tokens=max_tokens,
533
- temperature=0.0,
534
- )
535
- response_tokens = tokenizer(completion.choices[0].text,
536
- add_special_tokens=False)["input_ids"]
537
- first_response = completion.choices[0].text
538
- completion = await client.completions.create(
539
- model=MODEL_NAME,
540
- prompt=prompt,
541
- max_tokens=max_tokens,
542
- temperature=0.0,
543
- logit_bias={str(token): -100
544
- for token in response_tokens},
545
- )
546
- assert first_response != completion.choices[0].text
547
-
548
-
549
-@pytest.mark.asyncio
550
-async def test_allowed_token_ids(client: openai.AsyncOpenAI):
551
- prompt = "Hello, my name is"
552
- max_tokens = 1
553
- tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
554
-
555
- # Test exclusive selection
556
- allowed_ids = [21555, 21557, 21558]
557
- completion = await client.completions.create(
558
- model=MODEL_NAME,
559
- prompt=prompt,
560
- max_tokens=max_tokens,
561
- temperature=0.0,
562
- seed=42,
563
- extra_body=dict(allowed_token_ids=allowed_ids),
564
- logprobs=1,
565
- )
566
- response_tokens = completion.choices[0].logprobs.tokens
567
- assert len(response_tokens) == 1
568
- assert tokenizer.convert_tokens_to_ids(response_tokens)[0] in allowed_ids
569
-
570
-
571
-@pytest.mark.asyncio
572
-@pytest.mark.parametrize("guided_decoding_backend",
573
- ["outlines", "lm-format-enforcer"])
574
-async def test_guided_json_completion(client: openai.AsyncOpenAI,
575
- guided_decoding_backend: str,
576
- sample_json_schema):
577
- completion = await client.completions.create(
578
- model=MODEL_NAME,
579
- prompt=f"Give an example JSON for an employee profile "
580
- f"that fits this schema: {sample_json_schema}",
581
- n=3,
582
- temperature=1.0,
583
- max_tokens=500,
584
- extra_body=dict(guided_json=sample_json_schema,
585
- guided_decoding_backend=guided_decoding_backend))
586
-
587
- assert completion.id is not None
588
- assert len(completion.choices) == 3
589
- for i in range(3):
590
- output_json = json.loads(completion.choices[i].text)
591
- jsonschema.validate(instance=output_json, schema=sample_json_schema)
592
-
593
-
594
-@pytest.mark.asyncio
595
-@pytest.mark.parametrize("guided_decoding_backend",
596
- ["outlines", "lm-format-enforcer"])
597
-async def test_guided_regex_completion(client: openai.AsyncOpenAI,
598
- guided_decoding_backend: str,
599
- sample_regex):
600
- completion = await client.completions.create(
601
- model=MODEL_NAME,
602
- prompt=f"Give an example IPv4 address with this regex: {sample_regex}",
603
- n=3,
604
- temperature=1.0,
605
- max_tokens=20,
606
- extra_body=dict(guided_regex=sample_regex,
607
- guided_decoding_backend=guided_decoding_backend))
608
-
609
- assert completion.id is not None
610
- assert len(completion.choices) == 3
611
- for i in range(3):
612
- assert re.fullmatch(sample_regex,
613
- completion.choices[i].text) is not None
614
-
615
-
616
-@pytest.mark.asyncio
617
-@pytest.mark.parametrize("guided_decoding_backend",
618
- ["outlines", "lm-format-enforcer"])
619
-async def test_guided_choice_completion(client: openai.AsyncOpenAI,
620
- guided_decoding_backend: str,
621
- sample_guided_choice):
622
- completion = await client.completions.create(
623
- model=MODEL_NAME,
624
- prompt="The best language for type-safe systems programming is ",
625
- n=2,
626
- temperature=1.0,
627
- max_tokens=10,
628
- extra_body=dict(guided_choice=sample_guided_choice,
629
- guided_decoding_backend=guided_decoding_backend))
630
-
631
- assert completion.id is not None
632
- assert len(completion.choices) == 2
633
- for i in range(2):
634
- assert completion.choices[i].text in sample_guided_choice
635
-
636
-
637
-@pytest.mark.asyncio
638
-async def test_guided_grammar(client: openai.AsyncOpenAI,
639
- sample_sql_statements):
640
-
641
- completion = await client.completions.create(
642
- model=MODEL_NAME,
643
- prompt=("Generate a sql state that select col_1 from "
644
- "table_1 where it is equals to 1"),
645
- temperature=1.0,
646
- max_tokens=500,
647
- extra_body=dict(guided_grammar=sample_sql_statements))
648
-
649
- content = completion.choices[0].text
650
-
651
- # use Lark to parse the output, and make sure it's a valid parse tree
652
- from lark import Lark
653
- parser = Lark(sample_sql_statements)
654
- parser.parse(content)
655
-
656
- # remove spaces for comparison b/c we removed them in the grammar
657
- ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(" ", "")
658
-
659
- assert content.strip() == ground_truth
660
-
661
-
662
-@pytest.mark.asyncio
663
-@pytest.mark.parametrize(
664
- # first test base model, then test loras
665
- "model_name",
666
- [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
667
-)
668
-@pytest.mark.parametrize("logprobs_arg", [1, 0])
669
-async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
670
- model_name: str, logprobs_arg: int):
671
- tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
672
- # test using text and token IDs
673
- for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
674
- completion = await client.completions.create(model=model_name,
675
- prompt=prompt,
676
- max_tokens=5,
677
- temperature=0.0,
678
- echo=True,
679
- logprobs=logprobs_arg)
680
-
681
- prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
682
- list) else prompt
683
- assert re.search(r"^" + prompt_text, completion.choices[0].text)
684
- logprobs = completion.choices[0].logprobs
685
- assert logprobs is not None
686
- assert len(logprobs.text_offset) > 5
687
- assert (len(logprobs.token_logprobs) > 5
688
- and logprobs.token_logprobs[0] is None)
689
- assert (len(logprobs.top_logprobs) > 5
690
- and logprobs.top_logprobs[0] is None)
691
- for top_logprobs in logprobs.top_logprobs[1:]:
692
- assert max(logprobs_arg,
693
- 1) <= len(top_logprobs) <= logprobs_arg + 1
694
- assert len(logprobs.tokens) > 5
695
-
696
-
697
-@pytest.mark.asyncio
698
-@pytest.mark.parametrize("guided_decoding_backend",
699
- ["outlines", "lm-format-enforcer"])
700
-async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
701
- guided_decoding_backend: str,
702
- sample_json_schema, sample_regex):
703
- with pytest.raises(openai.BadRequestError):
704
- _ = await client.completions.create(
705
- model=MODEL_NAME,
706
- prompt="Give an example JSON that fits this schema: 42",
707
- extra_body=dict(guided_json=42,
708
- guided_decoding_backend=guided_decoding_backend))
709
-
710
- with pytest.raises(openai.BadRequestError):
711
- _ = await client.completions.create(
712
- model=MODEL_NAME,
713
- prompt="Give an example string that fits this regex",
714
- extra_body=dict(guided_regex=sample_regex,
715
- guided_json=sample_json_schema))