@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dfb1a15dcb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[ci][frontend] deduplicate tests (#7101)

youkaichao committed Aug 5, 2024 at 15:59 UTC dfb1a15dcb4c24bf7ff0ba7ddfc5d623ad519d7d
2 files changed +6 -723
tests/entrypoints/openai/test_completion.py
+6 -8
@@ -87,15 +87,13 @@ def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files,
87 ]
88
89
90 -@pytest.fixture(scope="module")
91 -def server(default_server_args):
90 +@pytest.fixture(scope="module",
91 + params=["", "--disable-frontend-multiprocessing"])
92 +def client(default_server_args, request):
93 + if request.param:
94 + default_server_args.append(request.param)
95 with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
93 - yield remote_server
94 -
95 -
96 -@pytest.fixture(scope="module")
97 -def client(server):
98 - return server.get_async_client()
96 + yield remote_server.get_async_client()
97
98
99 @pytest.mark.asyncio
tests/entrypoints/openai/test_disable_mp.py deleted
-715
@@ -1,715 +0,0 @@
1 -"""
2 -Repeat of tests in test_completion.py with the non-mp backend.
3 -"""
4 -
5 -# imports for guided decoding tests
6 -import json
7 -import re
8 -import shutil
9 -from tempfile import TemporaryDirectory
10 -from typing import List
11 -
12 -import jsonschema
13 -import openai # use the official client for correctness check
14 -import pytest
15 -# downloading lora to test lora requests
16 -from huggingface_hub import snapshot_download
17 -from openai import BadRequestError
18 -from transformers import AutoTokenizer
19 -
20 -from vllm.transformers_utils.tokenizer import get_tokenizer
21 -
22 -from ...utils import RemoteOpenAIServer
23 -
24 -# any model with a chat template should work here
25 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
26 -# technically these adapters use a different base model,
27 -# but we're not testing generation quality here
28 -LORA_NAME = "typeof/zephyr-7b-beta-lora"
29 -PA_NAME = "swapnilbp/llama_tweet_ptune"
30 -# if PA_NAME changes, PA_NUM_VIRTUAL_TOKENS might also
31 -# need to change to match the prompt adapter
32 -PA_NUM_VIRTUAL_TOKENS = 8
33 -
34 -
35 -@pytest.fixture(scope="module")
36 -def zephyr_lora_files():
37 - return snapshot_download(repo_id=LORA_NAME)
38 -
39 -
40 -@pytest.fixture(scope="module")
41 -def zephyr_lora_added_tokens_files(zephyr_lora_files):
42 - tmp_dir = TemporaryDirectory()
43 - tmp_model_dir = f"{tmp_dir.name}/zephyr"
44 - shutil.copytree(zephyr_lora_files, tmp_model_dir)
45 - tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
46 - # Copy tokenizer to adapter and add some unique tokens
47 - # 32000, 32001, 32002
48 - added = tokenizer.add_tokens(["vllm1", "vllm2", "vllm3"],
49 - special_tokens=True)
50 - assert added == 3
51 - tokenizer.save_pretrained(tmp_model_dir)
52 - yield tmp_model_dir
53 - tmp_dir.cleanup()
54 -
55 -
56 -@pytest.fixture(scope="module")
57 -def zephyr_pa_files():
58 - return snapshot_download(repo_id=PA_NAME)
59 -
60 -
61 -@pytest.fixture(scope="module")
62 -def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files,
63 - zephyr_pa_files):
64 - return [
65 - # use half precision for speed and memory savings in CI environment
66 - "--dtype",
67 - "bfloat16",
68 - "--max-model-len",
69 - "8192",
70 - "--max-num-seqs",
71 - "128",
72 - "--enforce-eager",
73 - # lora config
74 - "--enable-lora",
75 - "--lora-modules",
76 - f"zephyr-lora={zephyr_lora_files}",
77 - f"zephyr-lora2={zephyr_lora_added_tokens_files}",
78 - "--max-lora-rank",
79 - "64",
80 - "--max-cpu-loras",
81 - "2",
82 - # pa config
83 - "--enable-prompt-adapter",
84 - "--prompt-adapters",
85 - f"zephyr-pa={zephyr_pa_files}",
86 - f"zephyr-pa2={zephyr_pa_files}",
87 - "--max-prompt-adapters",
88 - "2",
89 - "--max-prompt-adapter-token",
90 - "128",
91 - "--disable-frontend-multiprocessing"
92 - ]
93 -
94 -
95 -@pytest.fixture(scope="module")
96 -def server(default_server_args):
97 - with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
98 - yield remote_server
99 -
100 -
101 -@pytest.fixture(scope="module")
102 -def client(server):
103 - return server.get_async_client()
104 -
105 -
106 -@pytest.mark.asyncio
107 -@pytest.mark.parametrize(
108 - # first test base model, then test loras, then test prompt adapters
109 - "model_name,num_virtual_tokens",
110 - [(MODEL_NAME, 0), ("zephyr-lora", 0), ("zephyr-lora2", 0),
111 - ("zephyr-pa", PA_NUM_VIRTUAL_TOKENS),
112 - ("zephyr-pa2", PA_NUM_VIRTUAL_TOKENS)],
113 -)
114 -async def test_single_completion(client: openai.AsyncOpenAI, model_name: str,
115 - num_virtual_tokens: int):
116 - completion = await client.completions.create(model=model_name,
117 - prompt="Hello, my name is",
118 - max_tokens=5,
119 - temperature=0.0)
120 -
121 - assert completion.id is not None
122 - assert completion.choices is not None and len(completion.choices) == 1
123 -
124 - choice = completion.choices[0]
125 - assert len(choice.text) >= 5
126 - assert choice.finish_reason == "length"
127 - assert completion.usage == openai.types.CompletionUsage(
128 - completion_tokens=5,
129 - prompt_tokens=6 + num_virtual_tokens,
130 - total_tokens=11 + num_virtual_tokens)
131 -
132 - # test using token IDs
133 - completion = await client.completions.create(
134 - model=model_name,
135 - prompt=[0, 0, 0, 0, 0],
136 - max_tokens=5,
137 - temperature=0.0,
138 - )
139 - assert len(completion.choices[0].text) >= 1
140 -
141 -
142 -@pytest.mark.asyncio
143 -async def test_added_lora_tokens(client: openai.AsyncOpenAI):
144 - # test using token IDs
145 - completion = await client.completions.create(
146 - model="zephyr-lora2",
147 - prompt=[0, 0, 32000, 32001, 32002],
148 - echo=True,
149 - max_tokens=5,
150 - temperature=0.0,
151 - )
152 - # Added tokens should appear in tokenized prompt
153 - assert completion.choices[0].text.startswith("<unk><unk>vllm1vllm2vllm3")
154 -
155 -
156 -@pytest.mark.asyncio
157 -async def test_added_lora_tokens_base_model(client: openai.AsyncOpenAI):
158 - # test using token IDs
159 - completion = await client.completions.create(
160 - model=MODEL_NAME,
161 - prompt=[0, 0, 32000, 32001, 32002],
162 - echo=True,
163 - max_tokens=5,
164 - temperature=0.0,
165 - )
166 - # Added tokens should not appear in tokenized prompt
167 - assert "vllm" not in completion.choices[0].text
168 -
169 -
170 -@pytest.mark.asyncio
171 -@pytest.mark.parametrize(
172 - # first test base model, then test loras, then test prompt adapters
173 - "model_name",
174 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2", "zephyr-pa", "zephyr-pa2"],
175 -)
176 -async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
177 - # test using token IDs
178 - completion = await client.completions.create(
179 - model=model_name,
180 - prompt=[0, 0, 0, 0, 0],
181 - max_tokens=5,
182 - temperature=0.0,
183 - logprobs=None,
184 - )
185 - choice = completion.choices[0]
186 - assert choice.logprobs is None
187 -
188 -
189 -@pytest.mark.asyncio
190 -@pytest.mark.parametrize(
191 - # just test 1 lora and 1 pa hereafter
192 - "model_name",
193 - [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
194 -)
195 -async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
196 - # test using token IDs
197 - completion = await client.completions.create(
198 - model=model_name,
199 - prompt=[0, 0, 0, 0, 0],
200 - max_tokens=5,
201 - temperature=0.0,
202 - logprobs=0,
203 - )
204 - choice = completion.choices[0]
205 - assert choice.logprobs is not None
206 - assert choice.logprobs.token_logprobs is not None
207 - assert choice.logprobs.top_logprobs is not None
208 - assert len(choice.logprobs.top_logprobs[0]) == 1
209 -
210 -
211 -@pytest.mark.asyncio
212 -@pytest.mark.parametrize(
213 - "model_name",
214 - [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
215 -)
216 -async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
217 - # test using token IDs
218 - completion = await client.completions.create(
219 - model=model_name,
220 - prompt=[0, 0, 0, 0, 0],
221 - max_tokens=5,
222 - temperature=0.0,
223 - logprobs=5,
224 - )
225 - choice = completion.choices[0]
226 - assert choice.logprobs is not None
227 - assert choice.logprobs.token_logprobs is not None
228 - assert choice.logprobs.top_logprobs is not None
229 - assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
230 -
231 -
232 -@pytest.mark.asyncio
233 -@pytest.mark.parametrize(
234 - "model_name",
235 - [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
236 -)
237 -async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
238 - model_name: str):
239 -
240 - with pytest.raises(
241 - (openai.BadRequestError, openai.APIError)): # test using token IDs
242 - await client.completions.create(
243 - model=model_name,
244 - prompt=[0, 0, 0, 0, 0],
245 - max_tokens=5,
246 - temperature=0.0,
247 - # vLLM has higher default max_logprobs (20 instead of 5) to support
248 - # both Completion API and Chat Completion API
249 - logprobs=21,
250 - )
251 - ...
252 - with pytest.raises(
253 - (openai.BadRequestError, openai.APIError)): # test using token IDs
254 - stream = await client.completions.create(
255 - model=model_name,
256 - prompt=[0, 0, 0, 0, 0],
257 - max_tokens=5,
258 - temperature=0.0,
259 - # vLLM has higher default max_logprobs (20 instead of 5) to support
260 - # both Completion API and Chat Completion API
261 - logprobs=30,
262 - stream=True,
263 - )
264 - async for chunk in stream:
265 - ...
266 -
267 - # the server should still work afterwards
268 - completion = await client.completions.create(
269 - model=model_name,
270 - prompt=[0, 0, 0, 0, 0],
271 - max_tokens=5,
272 - temperature=0.0,
273 - )
274 - assert len(completion.choices[0].text) >= 0
275 -
276 -
277 -@pytest.mark.asyncio
278 -@pytest.mark.parametrize(
279 - "model_name",
280 - [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
281 -)
282 -async def test_completion_streaming(client: openai.AsyncOpenAI,
283 - model_name: str):
284 - prompt = "What is an LLM?"
285 -
286 - single_completion = await client.completions.create(
287 - model=model_name,
288 - prompt=prompt,
289 - max_tokens=5,
290 - temperature=0.0,
291 - )
292 - single_output = single_completion.choices[0].text
293 - stream = await client.completions.create(model=model_name,
294 - prompt=prompt,
295 - max_tokens=5,
296 - temperature=0.0,
297 - stream=True)
298 - chunks: List[str] = []
299 - finish_reason_count = 0
300 - async for chunk in stream:
301 - chunks.append(chunk.choices[0].text)
302 - if chunk.choices[0].finish_reason is not None:
303 - finish_reason_count += 1
304 - # finish reason should only return in last block
305 - assert finish_reason_count == 1
306 - assert chunk.choices[0].finish_reason == "length"
307 - assert chunk.choices[0].text
308 - assert "".join(chunks) == single_output
309 -
310 -
311 -@pytest.mark.asyncio
312 -@pytest.mark.parametrize(
313 - "model_name",
314 - [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
315 -)
316 -async def test_completion_stream_options(client: openai.AsyncOpenAI,
317 - model_name: str):
318 - prompt = "What is the capital of France?"
319 -
320 - # Test stream=True, stream_options=
321 - # {"include_usage": False, "continuous_usage_stats": False}
322 - stream = await client.completions.create(model=model_name,
323 - prompt=prompt,
324 - max_tokens=5,
325 - temperature=0.0,
326 - stream=True,
327 - stream_options={
328 - "include_usage": False,
329 - "continuous_usage_stats":
330 - False,
331 - })
332 -
333 - async for chunk in stream:
334 - assert chunk.usage is None
335 -
336 - # Test stream=True, stream_options=
337 - # {"include_usage": False, "continuous_usage_stats": True}
338 - stream = await client.completions.create(model=model_name,
339 - prompt=prompt,
340 - max_tokens=5,
341 - temperature=0.0,
342 - stream=True,
343 - stream_options={
344 - "include_usage": False,
345 - "continuous_usage_stats":
346 - True,
347 - })
348 - async for chunk in stream:
349 - assert chunk.usage is None
350 -
351 - # Test stream=True, stream_options=
352 - # {"include_usage": True, "continuous_usage_stats": False}
353 - stream = await client.completions.create(model=model_name,
354 - prompt=prompt,
355 - max_tokens=5,
356 - temperature=0.0,
357 - stream=True,
358 - stream_options={
359 - "include_usage": True,
360 - "continuous_usage_stats":
361 - False,
362 - })
363 - async for chunk in stream:
364 - if chunk.choices[0].finish_reason is None:
365 - assert chunk.usage is None
366 - else:
367 - assert chunk.usage is None
368 - final_chunk = await stream.__anext__()
369 - assert final_chunk.usage is not None
370 - assert final_chunk.usage.prompt_tokens > 0
371 - assert final_chunk.usage.completion_tokens > 0
372 - assert final_chunk.usage.total_tokens == (
373 - final_chunk.usage.prompt_tokens +
374 - final_chunk.usage.completion_tokens)
375 - assert final_chunk.choices == []
376 -
377 - # Test stream=True, stream_options=
378 - # {"include_usage": True, "continuous_usage_stats": True}
379 - stream = await client.completions.create(model=model_name,
380 - prompt=prompt,
381 - max_tokens=5,
382 - temperature=0.0,
383 - stream=True,
384 - stream_options={
385 - "include_usage": True,
386 - "continuous_usage_stats":
387 - True,
388 - })
389 - async for chunk in stream:
390 - assert chunk.usage is not None
391 - assert chunk.usage.prompt_tokens > 0
392 - assert chunk.usage.completion_tokens > 0
393 - assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
394 - chunk.usage.completion_tokens)
395 - if chunk.choices[0].finish_reason is not None:
396 - final_chunk = await stream.__anext__()
397 - assert final_chunk.usage is not None
398 - assert final_chunk.usage.prompt_tokens > 0
399 - assert final_chunk.usage.completion_tokens > 0
400 - assert final_chunk.usage.total_tokens == (
401 - final_chunk.usage.prompt_tokens +
402 - final_chunk.usage.completion_tokens)
403 - assert final_chunk.choices == []
404 -
405 - # Test stream=False, stream_options=
406 - # {"include_usage": None}
407 - with pytest.raises(BadRequestError):
408 - await client.completions.create(model=model_name,
409 - prompt=prompt,
410 - max_tokens=5,
411 - temperature=0.0,
412 - stream=False,
413 - stream_options={"include_usage": None})
414 -
415 - # Test stream=False, stream_options=
416 - # {"include_usage": True}
417 - with pytest.raises(BadRequestError):
418 - await client.completions.create(model=model_name,
419 - prompt=prompt,
420 - max_tokens=5,
421 - temperature=0.0,
422 - stream=False,
423 - stream_options={"include_usage": True})
424 -
425 - # Test stream=False, stream_options=
426 - # {"continuous_usage_stats": None}
427 - with pytest.raises(BadRequestError):
428 - await client.completions.create(
429 - model=model_name,
430 - prompt=prompt,
431 - max_tokens=5,
432 - temperature=0.0,
433 - stream=False,
434 - stream_options={"continuous_usage_stats": None})
435 -
436 - # Test stream=False, stream_options=
437 - # {"continuous_usage_stats": True}
438 - with pytest.raises(BadRequestError):
439 - await client.completions.create(
440 - model=model_name,
441 - prompt=prompt,
442 - max_tokens=5,
443 - temperature=0.0,
444 - stream=False,
445 - stream_options={"continuous_usage_stats": True})
446 -
447 -
448 -@pytest.mark.asyncio
449 -@pytest.mark.parametrize(
450 - "model_name",
451 - [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
452 -)
453 -async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
454 - # test both text and token IDs
455 - for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
456 - # test simple list
457 - batch = await client.completions.create(
458 - model=model_name,
459 - prompt=prompts,
460 - max_tokens=5,
461 - temperature=0.0,
462 - )
463 - assert len(batch.choices) == 2
464 - assert batch.choices[0].text == batch.choices[1].text
465 -
466 - # test n = 2
467 - batch = await client.completions.create(
468 - model=model_name,
469 - prompt=prompts,
470 - n=2,
471 - max_tokens=5,
472 - temperature=0.0,
473 - extra_body=dict(
474 - # NOTE: this has to be true for n > 1 in vLLM, but not necessary
475 - # for official client.
476 - use_beam_search=True),
477 - )
478 - assert len(batch.choices) == 4
479 - assert batch.choices[0].text != batch.choices[
480 - 1].text, "beam search should be different"
481 - assert batch.choices[0].text == batch.choices[
482 - 2].text, "two copies of the same prompt should be the same"
483 - assert batch.choices[1].text == batch.choices[
484 - 3].text, "two copies of the same prompt should be the same"
485 -
486 - # test streaming
487 - batch = await client.completions.create(
488 - model=model_name,
489 - prompt=prompts,
490 - max_tokens=5,
491 - temperature=0.0,
492 - stream=True,
493 - )
494 - texts = [""] * 2
495 - async for chunk in batch:
496 - assert len(chunk.choices) == 1
497 - choice = chunk.choices[0]
498 - texts[choice.index] += choice.text
499 - assert texts[0] == texts[1]
500 -
501 -
502 -@pytest.mark.asyncio
503 -async def test_logits_bias(client: openai.AsyncOpenAI):
504 - prompt = "Hello, my name is"
505 - max_tokens = 5
506 - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
507 -
508 - # Test exclusive selection
509 - token_id = 1000
510 - completion = await client.completions.create(
511 - model=MODEL_NAME,
512 - prompt=prompt,
513 - max_tokens=max_tokens,
514 - temperature=0.0,
515 - logit_bias={str(token_id): 100},
516 - seed=42,
517 - )
518 - assert len(completion.choices[0].text) >= 5
519 - response_tokens = tokenizer(completion.choices[0].text,
520 - add_special_tokens=False)["input_ids"]
521 - expected_tokens = tokenizer(tokenizer.decode([token_id] * 5),
522 - add_special_tokens=False)["input_ids"]
523 - assert all([
524 - response == expected
525 - for response, expected in zip(response_tokens, expected_tokens)
526 - ])
527 -
528 - # Test ban
529 - completion = await client.completions.create(
530 - model=MODEL_NAME,
531 - prompt=prompt,
532 - max_tokens=max_tokens,
533 - temperature=0.0,
534 - )
535 - response_tokens = tokenizer(completion.choices[0].text,
536 - add_special_tokens=False)["input_ids"]
537 - first_response = completion.choices[0].text
538 - completion = await client.completions.create(
539 - model=MODEL_NAME,
540 - prompt=prompt,
541 - max_tokens=max_tokens,
542 - temperature=0.0,
543 - logit_bias={str(token): -100
544 - for token in response_tokens},
545 - )
546 - assert first_response != completion.choices[0].text
547 -
548 -
549 -@pytest.mark.asyncio
550 -async def test_allowed_token_ids(client: openai.AsyncOpenAI):
551 - prompt = "Hello, my name is"
552 - max_tokens = 1
553 - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
554 -
555 - # Test exclusive selection
556 - allowed_ids = [21555, 21557, 21558]
557 - completion = await client.completions.create(
558 - model=MODEL_NAME,
559 - prompt=prompt,
560 - max_tokens=max_tokens,
561 - temperature=0.0,
562 - seed=42,
563 - extra_body=dict(allowed_token_ids=allowed_ids),
564 - logprobs=1,
565 - )
566 - response_tokens = completion.choices[0].logprobs.tokens
567 - assert len(response_tokens) == 1
568 - assert tokenizer.convert_tokens_to_ids(response_tokens)[0] in allowed_ids
569 -
570 -
571 -@pytest.mark.asyncio
572 -@pytest.mark.parametrize("guided_decoding_backend",
573 - ["outlines", "lm-format-enforcer"])
574 -async def test_guided_json_completion(client: openai.AsyncOpenAI,
575 - guided_decoding_backend: str,
576 - sample_json_schema):
577 - completion = await client.completions.create(
578 - model=MODEL_NAME,
579 - prompt=f"Give an example JSON for an employee profile "
580 - f"that fits this schema: {sample_json_schema}",
581 - n=3,
582 - temperature=1.0,
583 - max_tokens=500,
584 - extra_body=dict(guided_json=sample_json_schema,
585 - guided_decoding_backend=guided_decoding_backend))
586 -
587 - assert completion.id is not None
588 - assert len(completion.choices) == 3
589 - for i in range(3):
590 - output_json = json.loads(completion.choices[i].text)
591 - jsonschema.validate(instance=output_json, schema=sample_json_schema)
592 -
593 -
594 -@pytest.mark.asyncio
595 -@pytest.mark.parametrize("guided_decoding_backend",
596 - ["outlines", "lm-format-enforcer"])
597 -async def test_guided_regex_completion(client: openai.AsyncOpenAI,
598 - guided_decoding_backend: str,
599 - sample_regex):
600 - completion = await client.completions.create(
601 - model=MODEL_NAME,
602 - prompt=f"Give an example IPv4 address with this regex: {sample_regex}",
603 - n=3,
604 - temperature=1.0,
605 - max_tokens=20,
606 - extra_body=dict(guided_regex=sample_regex,
607 - guided_decoding_backend=guided_decoding_backend))
608 -
609 - assert completion.id is not None
610 - assert len(completion.choices) == 3
611 - for i in range(3):
612 - assert re.fullmatch(sample_regex,
613 - completion.choices[i].text) is not None
614 -
615 -
616 -@pytest.mark.asyncio
617 -@pytest.mark.parametrize("guided_decoding_backend",
618 - ["outlines", "lm-format-enforcer"])
619 -async def test_guided_choice_completion(client: openai.AsyncOpenAI,
620 - guided_decoding_backend: str,
621 - sample_guided_choice):
622 - completion = await client.completions.create(
623 - model=MODEL_NAME,
624 - prompt="The best language for type-safe systems programming is ",
625 - n=2,
626 - temperature=1.0,
627 - max_tokens=10,
628 - extra_body=dict(guided_choice=sample_guided_choice,
629 - guided_decoding_backend=guided_decoding_backend))
630 -
631 - assert completion.id is not None
632 - assert len(completion.choices) == 2
633 - for i in range(2):
634 - assert completion.choices[i].text in sample_guided_choice
635 -
636 -
637 -@pytest.mark.asyncio
638 -async def test_guided_grammar(client: openai.AsyncOpenAI,
639 - sample_sql_statements):
640 -
641 - completion = await client.completions.create(
642 - model=MODEL_NAME,
643 - prompt=("Generate a sql state that select col_1 from "
644 - "table_1 where it is equals to 1"),
645 - temperature=1.0,
646 - max_tokens=500,
647 - extra_body=dict(guided_grammar=sample_sql_statements))
648 -
649 - content = completion.choices[0].text
650 -
651 - # use Lark to parse the output, and make sure it's a valid parse tree
652 - from lark import Lark
653 - parser = Lark(sample_sql_statements)
654 - parser.parse(content)
655 -
656 - # remove spaces for comparison b/c we removed them in the grammar
657 - ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(" ", "")
658 -
659 - assert content.strip() == ground_truth
660 -
661 -
662 -@pytest.mark.asyncio
663 -@pytest.mark.parametrize(
664 - # first test base model, then test loras
665 - "model_name",
666 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
667 -)
668 -@pytest.mark.parametrize("logprobs_arg", [1, 0])
669 -async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
670 - model_name: str, logprobs_arg: int):
671 - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
672 - # test using text and token IDs
673 - for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
674 - completion = await client.completions.create(model=model_name,
675 - prompt=prompt,
676 - max_tokens=5,
677 - temperature=0.0,
678 - echo=True,
679 - logprobs=logprobs_arg)
680 -
681 - prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
682 - list) else prompt
683 - assert re.search(r"^" + prompt_text, completion.choices[0].text)
684 - logprobs = completion.choices[0].logprobs
685 - assert logprobs is not None
686 - assert len(logprobs.text_offset) > 5
687 - assert (len(logprobs.token_logprobs) > 5
688 - and logprobs.token_logprobs[0] is None)
689 - assert (len(logprobs.top_logprobs) > 5
690 - and logprobs.top_logprobs[0] is None)
691 - for top_logprobs in logprobs.top_logprobs[1:]:
692 - assert max(logprobs_arg,
693 - 1) <= len(top_logprobs) <= logprobs_arg + 1
694 - assert len(logprobs.tokens) > 5
695 -
696 -
697 -@pytest.mark.asyncio
698 -@pytest.mark.parametrize("guided_decoding_backend",
699 - ["outlines", "lm-format-enforcer"])
700 -async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
701 - guided_decoding_backend: str,
702 - sample_json_schema, sample_regex):
703 - with pytest.raises(openai.BadRequestError):
704 - _ = await client.completions.create(
705 - model=MODEL_NAME,
706 - prompt="Give an example JSON that fits this schema: 42",
707 - extra_body=dict(guided_json=42,
708 - guided_decoding_backend=guided_decoding_backend))
709 -
710 - with pytest.raises(openai.BadRequestError):
711 - _ = await client.completions.create(
712 - model=MODEL_NAME,
713 - prompt="Give an example string that fits this regex",
714 - extra_body=dict(guided_regex=sample_regex,
715 - guided_json=sample_json_schema))