@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3b752a6555
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[CI/Build] [2/3] Reorganize entrypoints tests (#5904)

Cyrus Leung committed Jun 28, 2024 at 22:59 UTC 3b752a6555a67bb4863ea821fc30fdda38b27355
4 files changed +653 -598
tests/entrypoints/test_llm_generate_multiple_loras.py
+1 -1
@@ -44,7 +44,7 @@ def llm():
44 cleanup()
45
46
47 -@pytest.fixture(scope="session")
47 +@pytest.fixture(scope="module")
48 def zephyr_lora_files():
49 return snapshot_download(repo_id=LORA_NAME)
50
tests/entrypoints/test_openai_chat.py
+1 -1
@@ -72,7 +72,7 @@ TEST_CHOICE = [
72 pytestmark = pytest.mark.openai
73
74
75 -@pytest.fixture(scope="session")
75 +@pytest.fixture(scope="module")
76 def zephyr_lora_files():
77 return snapshot_download(repo_id=LORA_NAME)
78
tests/entrypoints/test_openai_completion.py new
+650
@@ -0,0 +1,650 @@
1 +# imports for guided decoding tests
2 +import json
3 +import re
4 +from typing import List
5 +
6 +import jsonschema
7 +import openai # use the official client for correctness check
8 +import pytest
9 +# using Ray for overall ease of process management, parallel requests,
10 +# and debugging.
11 +import ray
12 +import requests
13 +# downloading lora to test lora requests
14 +from huggingface_hub import snapshot_download
15 +from openai import BadRequestError
16 +
17 +from vllm.transformers_utils.tokenizer import get_tokenizer
18 +
19 +from ..utils import RemoteOpenAIServer
20 +
21 +# any model with a chat template should work here
22 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
23 +# technically this needs Mistral-7B-v0.1 as base, but we're not testing
24 +# generation quality here
25 +LORA_NAME = "typeof/zephyr-7b-beta-lora"
26 +
27 +TEST_SCHEMA = {
28 + "type": "object",
29 + "properties": {
30 + "name": {
31 + "type": "string"
32 + },
33 + "age": {
34 + "type": "integer"
35 + },
36 + "skills": {
37 + "type": "array",
38 + "items": {
39 + "type": "string",
40 + "maxLength": 10
41 + },
42 + "minItems": 3
43 + },
44 + "work history": {
45 + "type": "array",
46 + "items": {
47 + "type": "object",
48 + "properties": {
49 + "company": {
50 + "type": "string"
51 + },
52 + "duration": {
53 + "type": "string"
54 + },
55 + "position": {
56 + "type": "string"
57 + }
58 + },
59 + "required": ["company", "position"]
60 + }
61 + }
62 + },
63 + "required": ["name", "age", "skills", "work history"]
64 +}
65 +
66 +TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
67 + r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
68 +
69 +TEST_CHOICE = [
70 + "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
71 + "Swift", "Kotlin"
72 +]
73 +
74 +pytestmark = pytest.mark.openai
75 +
76 +
77 +@pytest.fixture(scope="module")
78 +def zephyr_lora_files():
79 + return snapshot_download(repo_id=LORA_NAME)
80 +
81 +
82 +@pytest.fixture(scope="module")
83 +def ray_ctx():
84 + ray.init()
85 + yield
86 + ray.shutdown()
87 +
88 +
89 +@pytest.fixture(scope="module")
90 +def server(zephyr_lora_files, ray_ctx):
91 + return RemoteOpenAIServer([
92 + "--model",
93 + MODEL_NAME,
94 + # use half precision for speed and memory savings in CI environment
95 + "--dtype",
96 + "bfloat16",
97 + "--max-model-len",
98 + "8192",
99 + "--enforce-eager",
100 + # lora config below
101 + "--enable-lora",
102 + "--lora-modules",
103 + f"zephyr-lora={zephyr_lora_files}",
104 + f"zephyr-lora2={zephyr_lora_files}",
105 + "--max-lora-rank",
106 + "64",
107 + "--max-cpu-loras",
108 + "2",
109 + "--max-num-seqs",
110 + "128",
111 + ])
112 +
113 +
114 +@pytest.fixture(scope="module")
115 +def client(server):
116 + return server.get_async_client()
117 +
118 +
119 +@pytest.mark.asyncio
120 +@pytest.mark.parametrize(
121 + # first test base model, then test loras
122 + "model_name",
123 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
124 +)
125 +async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
126 + completion = await client.completions.create(model=model_name,
127 + prompt="Hello, my name is",
128 + max_tokens=5,
129 + temperature=0.0)
130 +
131 + assert completion.id is not None
132 + assert completion.choices is not None and len(completion.choices) == 1
133 +
134 + choice = completion.choices[0]
135 + assert len(choice.text) >= 5
136 + assert choice.finish_reason == "length"
137 + assert completion.usage == openai.types.CompletionUsage(
138 + completion_tokens=5, prompt_tokens=6, total_tokens=11)
139 +
140 + # test using token IDs
141 + completion = await client.completions.create(
142 + model=MODEL_NAME,
143 + prompt=[0, 0, 0, 0, 0],
144 + max_tokens=5,
145 + temperature=0.0,
146 + )
147 + assert len(completion.choices[0].text) >= 5
148 +
149 +
150 +@pytest.mark.asyncio
151 +@pytest.mark.parametrize(
152 + # first test base model, then test loras
153 + "model_name",
154 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
155 +)
156 +async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
157 + # test using token IDs
158 + completion = await client.completions.create(
159 + model=MODEL_NAME,
160 + prompt=[0, 0, 0, 0, 0],
161 + max_tokens=5,
162 + temperature=0.0,
163 + logprobs=None,
164 + )
165 + choice = completion.choices[0]
166 + assert choice.logprobs is None
167 +
168 +
169 +@pytest.mark.asyncio
170 +@pytest.mark.parametrize(
171 + # just test 1 lora hereafter
172 + "model_name",
173 + [MODEL_NAME, "zephyr-lora"],
174 +)
175 +async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
176 + # test using token IDs
177 + completion = await client.completions.create(
178 + model=MODEL_NAME,
179 + prompt=[0, 0, 0, 0, 0],
180 + max_tokens=5,
181 + temperature=0.0,
182 + logprobs=0,
183 + )
184 + choice = completion.choices[0]
185 + assert choice.logprobs is not None
186 + assert choice.logprobs.token_logprobs is not None
187 + assert choice.logprobs.top_logprobs is not None
188 + assert len(choice.logprobs.top_logprobs[0]) == 1
189 +
190 +
191 +@pytest.mark.asyncio
192 +@pytest.mark.parametrize(
193 + "model_name",
194 + [MODEL_NAME, "zephyr-lora"],
195 +)
196 +async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
197 + # test using token IDs
198 + completion = await client.completions.create(
199 + model=MODEL_NAME,
200 + prompt=[0, 0, 0, 0, 0],
201 + max_tokens=5,
202 + temperature=0.0,
203 + logprobs=5,
204 + )
205 + choice = completion.choices[0]
206 + assert choice.logprobs is not None
207 + assert choice.logprobs.token_logprobs is not None
208 + assert choice.logprobs.top_logprobs is not None
209 + assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
210 +
211 +
212 +@pytest.mark.asyncio
213 +@pytest.mark.parametrize(
214 + "model_name",
215 + [MODEL_NAME, "zephyr-lora"],
216 +)
217 +async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
218 + model_name: str):
219 +
220 + with pytest.raises(
221 + (openai.BadRequestError, openai.APIError)): # test using token IDs
222 + await client.completions.create(
223 + model=MODEL_NAME,
224 + prompt=[0, 0, 0, 0, 0],
225 + max_tokens=5,
226 + temperature=0.0,
227 + # vLLM has higher default max_logprobs (20 instead of 5) to support
228 + # both Completion API and Chat Completion API
229 + logprobs=21,
230 + )
231 + ...
232 + with pytest.raises(
233 + (openai.BadRequestError, openai.APIError)): # test using token IDs
234 + stream = await client.completions.create(
235 + model=MODEL_NAME,
236 + prompt=[0, 0, 0, 0, 0],
237 + max_tokens=5,
238 + temperature=0.0,
239 + # vLLM has higher default max_logprobs (20 instead of 5) to support
240 + # both Completion API and Chat Completion API
241 + logprobs=30,
242 + stream=True,
243 + )
244 + async for chunk in stream:
245 + ...
246 +
247 + # the server should still work afterwards
248 + completion = await client.completions.create(
249 + model=model_name,
250 + prompt=[0, 0, 0, 0, 0],
251 + max_tokens=5,
252 + temperature=0.0,
253 + )
254 + assert len(completion.choices[0].text) >= 0
255 +
256 +
257 +@pytest.mark.asyncio
258 +@pytest.mark.parametrize(
259 + "model_name",
260 + [MODEL_NAME, "zephyr-lora"],
261 +)
262 +async def test_completion_streaming(client: openai.AsyncOpenAI,
263 + model_name: str):
264 + prompt = "What is an LLM?"
265 +
266 + single_completion = await client.completions.create(
267 + model=model_name,
268 + prompt=prompt,
269 + max_tokens=5,
270 + temperature=0.0,
271 + )
272 + single_output = single_completion.choices[0].text
273 + stream = await client.completions.create(model=model_name,
274 + prompt=prompt,
275 + max_tokens=5,
276 + temperature=0.0,
277 + stream=True)
278 + chunks: List[str] = []
279 + finish_reason_count = 0
280 + async for chunk in stream:
281 + chunks.append(chunk.choices[0].text)
282 + if chunk.choices[0].finish_reason is not None:
283 + finish_reason_count += 1
284 + # finish reason should only return in last block
285 + assert finish_reason_count == 1
286 + assert chunk.choices[0].finish_reason == "length"
287 + assert chunk.choices[0].text
288 + assert "".join(chunks) == single_output
289 +
290 +
291 +@pytest.mark.asyncio
292 +@pytest.mark.parametrize(
293 + "model_name",
294 + ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
295 +)
296 +async def test_completion_stream_options(client: openai.AsyncOpenAI,
297 + model_name: str):
298 + prompt = "What is the capital of France?"
299 +
300 + # Test stream=True, stream_options={"include_usage": False}
301 + stream = await client.completions.create(
302 + model=model_name,
303 + prompt=prompt,
304 + max_tokens=5,
305 + temperature=0.0,
306 + stream=True,
307 + stream_options={"include_usage": False})
308 + async for chunk in stream:
309 + assert chunk.usage is None
310 +
311 + # Test stream=True, stream_options={"include_usage": True}
312 + stream = await client.completions.create(
313 + model=model_name,
314 + prompt=prompt,
315 + max_tokens=5,
316 + temperature=0.0,
317 + stream=True,
318 + stream_options={"include_usage": True})
319 + async for chunk in stream:
320 + if chunk.choices[0].finish_reason is None:
321 + assert chunk.usage is None
322 + else:
323 + assert chunk.usage is None
324 + final_chunk = await stream.__anext__()
325 + assert final_chunk.usage is not None
326 + assert final_chunk.usage.prompt_tokens > 0
327 + assert final_chunk.usage.completion_tokens > 0
328 + assert final_chunk.usage.total_tokens == (
329 + final_chunk.usage.prompt_tokens +
330 + final_chunk.usage.completion_tokens)
331 + assert final_chunk.choices == []
332 +
333 + # Test stream=False, stream_options={"include_usage": None}
334 + with pytest.raises(BadRequestError):
335 + await client.completions.create(model=model_name,
336 + prompt=prompt,
337 + max_tokens=5,
338 + temperature=0.0,
339 + stream=False,
340 + stream_options={"include_usage": None})
341 +
342 + # Test stream=False, stream_options={"include_usage": True}
343 + with pytest.raises(BadRequestError):
344 + await client.completions.create(model=model_name,
345 + prompt=prompt,
346 + max_tokens=5,
347 + temperature=0.0,
348 + stream=False,
349 + stream_options={"include_usage": True})
350 +
351 +
352 +@pytest.mark.asyncio
353 +@pytest.mark.parametrize(
354 + # just test 1 lora hereafter
355 + "model_name",
356 + [MODEL_NAME, "zephyr-lora"],
357 +)
358 +async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
359 + # test both text and token IDs
360 + for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
361 + # test simple list
362 + batch = await client.completions.create(
363 + model=model_name,
364 + prompt=prompts,
365 + max_tokens=5,
366 + temperature=0.0,
367 + )
368 + assert len(batch.choices) == 2
369 + assert batch.choices[0].text == batch.choices[1].text
370 +
371 + # test n = 2
372 + batch = await client.completions.create(
373 + model=model_name,
374 + prompt=prompts,
375 + n=2,
376 + max_tokens=5,
377 + temperature=0.0,
378 + extra_body=dict(
379 + # NOTE: this has to be true for n > 1 in vLLM, but not necessary
380 + # for official client.
381 + use_beam_search=True),
382 + )
383 + assert len(batch.choices) == 4
384 + assert batch.choices[0].text != batch.choices[
385 + 1].text, "beam search should be different"
386 + assert batch.choices[0].text == batch.choices[
387 + 2].text, "two copies of the same prompt should be the same"
388 + assert batch.choices[1].text == batch.choices[
389 + 3].text, "two copies of the same prompt should be the same"
390 +
391 + # test streaming
392 + batch = await client.completions.create(
393 + model=model_name,
394 + prompt=prompts,
395 + max_tokens=5,
396 + temperature=0.0,
397 + stream=True,
398 + )
399 + texts = [""] * 2
400 + async for chunk in batch:
401 + assert len(chunk.choices) == 1
402 + choice = chunk.choices[0]
403 + texts[choice.index] += choice.text
404 + assert texts[0] == texts[1]
405 +
406 +
407 +@pytest.mark.asyncio
408 +async def test_logits_bias(client: openai.AsyncOpenAI):
409 + prompt = "Hello, my name is"
410 + max_tokens = 5
411 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
412 +
413 + # Test exclusive selection
414 + token_id = 1000
415 + completion = await client.completions.create(
416 + model=MODEL_NAME,
417 + prompt=prompt,
418 + max_tokens=max_tokens,
419 + temperature=0.0,
420 + logit_bias={str(token_id): 100},
421 + seed=42,
422 + )
423 + assert len(completion.choices[0].text) >= 5
424 + response_tokens = tokenizer(completion.choices[0].text,
425 + add_special_tokens=False)["input_ids"]
426 + expected_tokens = tokenizer(tokenizer.decode([token_id] * 5),
427 + add_special_tokens=False)["input_ids"]
428 + assert all([
429 + response == expected
430 + for response, expected in zip(response_tokens, expected_tokens)
431 + ])
432 +
433 + # Test ban
434 + completion = await client.completions.create(
435 + model=MODEL_NAME,
436 + prompt=prompt,
437 + max_tokens=max_tokens,
438 + temperature=0.0,
439 + )
440 + response_tokens = tokenizer(completion.choices[0].text,
441 + add_special_tokens=False)["input_ids"]
442 + first_response = completion.choices[0].text
443 + completion = await client.completions.create(
444 + model=MODEL_NAME,
445 + prompt=prompt,
446 + max_tokens=max_tokens,
447 + temperature=0.0,
448 + logit_bias={str(token): -100
449 + for token in response_tokens},
450 + )
451 + assert first_response != completion.choices[0].text
452 +
453 +
454 +@pytest.mark.asyncio
455 +@pytest.mark.parametrize("guided_decoding_backend",
456 + ["outlines", "lm-format-enforcer"])
457 +async def test_guided_json_completion(client: openai.AsyncOpenAI,
458 + guided_decoding_backend: str):
459 + completion = await client.completions.create(
460 + model=MODEL_NAME,
461 + prompt=f"Give an example JSON for an employee profile "
462 + f"that fits this schema: {TEST_SCHEMA}",
463 + n=3,
464 + temperature=1.0,
465 + max_tokens=500,
466 + extra_body=dict(guided_json=TEST_SCHEMA,
467 + guided_decoding_backend=guided_decoding_backend))
468 +
469 + assert completion.id is not None
470 + assert len(completion.choices) == 3
471 + for i in range(3):
472 + output_json = json.loads(completion.choices[i].text)
473 + jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
474 +
475 +
476 +@pytest.mark.asyncio
477 +@pytest.mark.parametrize("guided_decoding_backend",
478 + ["outlines", "lm-format-enforcer"])
479 +async def test_guided_regex_completion(client: openai.AsyncOpenAI,
480 + guided_decoding_backend: str):
481 + completion = await client.completions.create(
482 + model=MODEL_NAME,
483 + prompt=f"Give an example IPv4 address with this regex: {TEST_REGEX}",
484 + n=3,
485 + temperature=1.0,
486 + max_tokens=20,
487 + extra_body=dict(guided_regex=TEST_REGEX,
488 + guided_decoding_backend=guided_decoding_backend))
489 +
490 + assert completion.id is not None
491 + assert len(completion.choices) == 3
492 + for i in range(3):
493 + assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
494 +
495 +
496 +@pytest.mark.asyncio
497 +@pytest.mark.parametrize("guided_decoding_backend",
498 + ["outlines", "lm-format-enforcer"])
499 +async def test_guided_choice_completion(client: openai.AsyncOpenAI,
500 + guided_decoding_backend: str):
501 + completion = await client.completions.create(
502 + model=MODEL_NAME,
503 + prompt="The best language for type-safe systems programming is ",
504 + n=2,
505 + temperature=1.0,
506 + max_tokens=10,
507 + extra_body=dict(guided_choice=TEST_CHOICE,
508 + guided_decoding_backend=guided_decoding_backend))
509 +
510 + assert completion.id is not None
511 + assert len(completion.choices) == 2
512 + for i in range(2):
513 + assert completion.choices[i].text in TEST_CHOICE
514 +
515 +
516 +@pytest.mark.asyncio
517 +async def test_guided_grammar(client: openai.AsyncOpenAI):
518 + simple_sql_grammar = """
519 +start: select_statement
520 +
521 +select_statement: "SELECT" column "from" table "where" condition
522 +
523 +column: "col_1" | "col_2"
524 +table: "table_1" | "table_2"
525 +condition: column "=" number
526 +
527 +number: "1" | "2"
528 +"""
529 +
530 + completion = await client.completions.create(
531 + model=MODEL_NAME,
532 + prompt=("Generate a sql state that select col_1 from "
533 + "table_1 where it is equals to 1"),
534 + temperature=1.0,
535 + max_tokens=500,
536 + extra_body=dict(guided_grammar=simple_sql_grammar))
537 +
538 + content = completion.choices[0].text
539 +
540 + # use Lark to parse the output, and make sure it's a valid parse tree
541 + from lark import Lark
542 + parser = Lark(simple_sql_grammar)
543 + parser.parse(content)
544 +
545 + # remove spaces for comparison b/c we removed them in the grammar
546 + ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(" ", "")
547 +
548 + assert content.strip() == ground_truth
549 +
550 +
551 +@pytest.mark.asyncio
552 +@pytest.mark.parametrize(
553 + # first test base model, then test loras
554 + "model_name",
555 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
556 +)
557 +@pytest.mark.parametrize("logprobs_arg", [1, 0])
558 +async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
559 + model_name: str, logprobs_arg: int):
560 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
561 + # test using text and token IDs
562 + for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
563 + completion = await client.completions.create(model=model_name,
564 + prompt=prompt,
565 + max_tokens=5,
566 + temperature=0.0,
567 + echo=True,
568 + logprobs=logprobs_arg)
569 +
570 + prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
571 + list) else prompt
572 + assert re.search(r"^" + prompt_text, completion.choices[0].text)
573 + logprobs = completion.choices[0].logprobs
574 + assert logprobs is not None
575 + assert len(logprobs.text_offset) > 5
576 + assert (len(logprobs.token_logprobs) > 5
577 + and logprobs.token_logprobs[0] is None)
578 + assert (len(logprobs.top_logprobs) > 5
579 + and logprobs.top_logprobs[0] is None)
580 + for top_logprobs in logprobs.top_logprobs[1:]:
581 + assert max(logprobs_arg,
582 + 1) <= len(top_logprobs) <= logprobs_arg + 1
583 + assert len(logprobs.tokens) > 5
584 +
585 +
586 +@pytest.mark.asyncio
587 +@pytest.mark.parametrize("guided_decoding_backend",
588 + ["outlines", "lm-format-enforcer"])
589 +async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
590 + guided_decoding_backend: str):
591 + with pytest.raises(openai.BadRequestError):
592 + _ = await client.completions.create(
593 + model=MODEL_NAME,
594 + prompt="Give an example JSON that fits this schema: 42",
595 + extra_body=dict(guided_json=42,
596 + guided_decoding_backend=guided_decoding_backend))
597 +
598 + with pytest.raises(openai.BadRequestError):
599 + _ = await client.completions.create(
600 + model=MODEL_NAME,
601 + prompt="Give an example string that fits this regex",
602 + extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
603 +
604 +
605 +@pytest.mark.asyncio
606 +@pytest.mark.parametrize(
607 + "model_name",
608 + [MODEL_NAME],
609 +)
610 +async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
611 + base_url = str(client.base_url)[:-3]
612 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
613 +
614 + for add_special in [False, True]:
615 + prompt = "This is a test prompt."
616 + tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
617 +
618 + response = requests.post(base_url + "/tokenize",
619 + json={
620 + "add_special_tokens": add_special,
621 + "model": model_name,
622 + "prompt": prompt
623 + })
624 + response.raise_for_status()
625 + assert response.json() == {
626 + "tokens": tokens,
627 + "count": len(tokens),
628 + "max_model_len": 8192
629 + }
630 +
631 +
632 +@pytest.mark.asyncio
633 +@pytest.mark.parametrize(
634 + "model_name",
635 + [MODEL_NAME],
636 +)
637 +async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
638 + base_url = str(client.base_url)[:-3]
639 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
640 +
641 + prompt = "This is a test prompt."
642 + tokens = tokenizer.encode(prompt, add_special_tokens=False)
643 +
644 + response = requests.post(base_url + "detokenize",
645 + json={
646 + "model": model_name,
647 + "tokens": tokens
648 + })
649 + response.raise_for_status()
650 + assert response.json() == {"prompt": prompt}
tests/entrypoints/test_openai_server.py
+1 -596
@@ -1,20 +1,10 @@
1 -# imports for guided decoding tests
2 -import json
3 -import re
4 -from typing import List
5 -
6 -import jsonschema
1 import openai # use the official client for correctness check
2 import pytest
3 # using Ray for overall ease of process management, parallel requests,
4 # and debugging.
5 import ray
12 -import requests
6 # downloading lora to test lora requests
7 from huggingface_hub import snapshot_download
15 -from openai import BadRequestError
16 -
17 -from vllm.transformers_utils.tokenizer import get_tokenizer
8
9 from ..utils import RemoteOpenAIServer
10
@@ -24,57 +14,10 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
14 # generation quality here
15 LORA_NAME = "typeof/zephyr-7b-beta-lora"
16
27 -TEST_SCHEMA = {
28 - "type": "object",
29 - "properties": {
30 - "name": {
31 - "type": "string"
32 - },
33 - "age": {
34 - "type": "integer"
35 - },
36 - "skills": {
37 - "type": "array",
38 - "items": {
39 - "type": "string",
40 - "maxLength": 10
41 - },
42 - "minItems": 3
43 - },
44 - "work history": {
45 - "type": "array",
46 - "items": {
47 - "type": "object",
48 - "properties": {
49 - "company": {
50 - "type": "string"
51 - },
52 - "duration": {
53 - "type": "string"
54 - },
55 - "position": {
56 - "type": "string"
57 - }
58 - },
59 - "required": ["company", "position"]
60 - }
61 - }
62 - },
63 - "required": ["name", "age", "skills", "work history"]
64 -}
65 -
66 -TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
67 - r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
68 -
69 -TEST_CHOICE = [
70 - "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
71 - "Swift", "Kotlin"
72 -]
73 -
17 pytestmark = pytest.mark.openai
18
19
77 -@pytest.fixture(scope="session")
20 +@pytest.fixture(scope="module")
21 def zephyr_lora_files():
22 return snapshot_download(repo_id=LORA_NAME)
23
@@ -126,541 +69,3 @@ async def test_check_models(client: openai.AsyncOpenAI):
69 assert all(model.root == MODEL_NAME for model in models)
70 assert lora_models[0].id == "zephyr-lora"
71 assert lora_models[1].id == "zephyr-lora2"
129 -
130 -
131 -@pytest.mark.asyncio
132 -@pytest.mark.parametrize(
133 - # first test base model, then test loras
134 - "model_name",
135 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
136 -)
137 -async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
138 - completion = await client.completions.create(model=model_name,
139 - prompt="Hello, my name is",
140 - max_tokens=5,
141 - temperature=0.0)
142 -
143 - assert completion.id is not None
144 - assert completion.choices is not None and len(completion.choices) == 1
145 -
146 - choice = completion.choices[0]
147 - assert len(choice.text) >= 5
148 - assert choice.finish_reason == "length"
149 - assert completion.usage == openai.types.CompletionUsage(
150 - completion_tokens=5, prompt_tokens=6, total_tokens=11)
151 -
152 - # test using token IDs
153 - completion = await client.completions.create(
154 - model=MODEL_NAME,
155 - prompt=[0, 0, 0, 0, 0],
156 - max_tokens=5,
157 - temperature=0.0,
158 - )
159 - assert len(completion.choices[0].text) >= 5
160 -
161 -
162 -@pytest.mark.asyncio
163 -@pytest.mark.parametrize(
164 - # first test base model, then test loras
165 - "model_name",
166 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
167 -)
168 -async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
169 - # test using token IDs
170 - completion = await client.completions.create(
171 - model=MODEL_NAME,
172 - prompt=[0, 0, 0, 0, 0],
173 - max_tokens=5,
174 - temperature=0.0,
175 - logprobs=None,
176 - )
177 - choice = completion.choices[0]
178 - assert choice.logprobs is None
179 -
180 -
181 -@pytest.mark.asyncio
182 -@pytest.mark.parametrize(
183 - # just test 1 lora hereafter
184 - "model_name",
185 - [MODEL_NAME, "zephyr-lora"],
186 -)
187 -async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
188 - # test using token IDs
189 - completion = await client.completions.create(
190 - model=MODEL_NAME,
191 - prompt=[0, 0, 0, 0, 0],
192 - max_tokens=5,
193 - temperature=0.0,
194 - logprobs=0,
195 - )
196 - choice = completion.choices[0]
197 - assert choice.logprobs is not None
198 - assert choice.logprobs.token_logprobs is not None
199 - assert choice.logprobs.top_logprobs is not None
200 - assert len(choice.logprobs.top_logprobs[0]) == 1
201 -
202 -
203 -@pytest.mark.asyncio
204 -@pytest.mark.parametrize(
205 - "model_name",
206 - [MODEL_NAME, "zephyr-lora"],
207 -)
208 -async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
209 - # test using token IDs
210 - completion = await client.completions.create(
211 - model=MODEL_NAME,
212 - prompt=[0, 0, 0, 0, 0],
213 - max_tokens=5,
214 - temperature=0.0,
215 - logprobs=5,
216 - )
217 - choice = completion.choices[0]
218 - assert choice.logprobs is not None
219 - assert choice.logprobs.token_logprobs is not None
220 - assert choice.logprobs.top_logprobs is not None
221 - assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
222 -
223 -
224 -@pytest.mark.asyncio
225 -@pytest.mark.parametrize(
226 - "model_name",
227 - [MODEL_NAME, "zephyr-lora"],
228 -)
229 -async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
230 - model_name: str):
231 -
232 - with pytest.raises(
233 - (openai.BadRequestError, openai.APIError)): # test using token IDs
234 - await client.completions.create(
235 - model=MODEL_NAME,
236 - prompt=[0, 0, 0, 0, 0],
237 - max_tokens=5,
238 - temperature=0.0,
239 - # vLLM has higher default max_logprobs (20 instead of 5) to support
240 - # both Completion API and Chat Completion API
241 - logprobs=21,
242 - )
243 - ...
244 - with pytest.raises(
245 - (openai.BadRequestError, openai.APIError)): # test using token IDs
246 - stream = await client.completions.create(
247 - model=MODEL_NAME,
248 - prompt=[0, 0, 0, 0, 0],
249 - max_tokens=5,
250 - temperature=0.0,
251 - # vLLM has higher default max_logprobs (20 instead of 5) to support
252 - # both Completion API and Chat Completion API
253 - logprobs=30,
254 - stream=True,
255 - )
256 - async for chunk in stream:
257 - ...
258 -
259 - # the server should still work afterwards
260 - completion = await client.completions.create(
261 - model=model_name,
262 - prompt=[0, 0, 0, 0, 0],
263 - max_tokens=5,
264 - temperature=0.0,
265 - )
266 - assert len(completion.choices[0].text) >= 0
267 -
268 -
269 -@pytest.mark.asyncio
270 -@pytest.mark.parametrize(
271 - "model_name",
272 - [MODEL_NAME, "zephyr-lora"],
273 -)
274 -async def test_completion_streaming(client: openai.AsyncOpenAI,
275 - model_name: str):
276 - prompt = "What is an LLM?"
277 -
278 - single_completion = await client.completions.create(
279 - model=model_name,
280 - prompt=prompt,
281 - max_tokens=5,
282 - temperature=0.0,
283 - )
284 - single_output = single_completion.choices[0].text
285 - stream = await client.completions.create(model=model_name,
286 - prompt=prompt,
287 - max_tokens=5,
288 - temperature=0.0,
289 - stream=True)
290 - chunks: List[str] = []
291 - finish_reason_count = 0
292 - async for chunk in stream:
293 - chunks.append(chunk.choices[0].text)
294 - if chunk.choices[0].finish_reason is not None:
295 - finish_reason_count += 1
296 - # finish reason should only return in last block
297 - assert finish_reason_count == 1
298 - assert chunk.choices[0].finish_reason == "length"
299 - assert chunk.choices[0].text
300 - assert "".join(chunks) == single_output
301 -
302 -
303 -@pytest.mark.asyncio
304 -@pytest.mark.parametrize(
305 - "model_name",
306 - ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
307 -)
308 -async def test_completion_stream_options(client: openai.AsyncOpenAI,
309 - model_name: str):
310 - prompt = "What is the capital of France?"
311 -
312 - # Test stream=True, stream_options={"include_usage": False}
313 - stream = await client.completions.create(
314 - model=model_name,
315 - prompt=prompt,
316 - max_tokens=5,
317 - temperature=0.0,
318 - stream=True,
319 - stream_options={"include_usage": False})
320 - async for chunk in stream:
321 - assert chunk.usage is None
322 -
323 - # Test stream=True, stream_options={"include_usage": True}
324 - stream = await client.completions.create(
325 - model=model_name,
326 - prompt=prompt,
327 - max_tokens=5,
328 - temperature=0.0,
329 - stream=True,
330 - stream_options={"include_usage": True})
331 - async for chunk in stream:
332 - if chunk.choices[0].finish_reason is None:
333 - assert chunk.usage is None
334 - else:
335 - assert chunk.usage is None
336 - final_chunk = await stream.__anext__()
337 - assert final_chunk.usage is not None
338 - assert final_chunk.usage.prompt_tokens > 0
339 - assert final_chunk.usage.completion_tokens > 0
340 - assert final_chunk.usage.total_tokens == (
341 - final_chunk.usage.prompt_tokens +
342 - final_chunk.usage.completion_tokens)
343 - assert final_chunk.choices == []
344 -
345 - # Test stream=False, stream_options={"include_usage": None}
346 - with pytest.raises(BadRequestError):
347 - await client.completions.create(model=model_name,
348 - prompt=prompt,
349 - max_tokens=5,
350 - temperature=0.0,
351 - stream=False,
352 - stream_options={"include_usage": None})
353 -
354 - # Test stream=False, stream_options={"include_usage": True}
355 - with pytest.raises(BadRequestError):
356 - await client.completions.create(model=model_name,
357 - prompt=prompt,
358 - max_tokens=5,
359 - temperature=0.0,
360 - stream=False,
361 - stream_options={"include_usage": True})
362 -
363 -
364 -@pytest.mark.asyncio
365 -@pytest.mark.parametrize(
366 - # just test 1 lora hereafter
367 - "model_name",
368 - [MODEL_NAME, "zephyr-lora"],
369 -)
370 -async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
371 - # test both text and token IDs
372 - for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
373 - # test simple list
374 - batch = await client.completions.create(
375 - model=model_name,
376 - prompt=prompts,
377 - max_tokens=5,
378 - temperature=0.0,
379 - )
380 - assert len(batch.choices) == 2
381 - assert batch.choices[0].text == batch.choices[1].text
382 -
383 - # test n = 2
384 - batch = await client.completions.create(
385 - model=model_name,
386 - prompt=prompts,
387 - n=2,
388 - max_tokens=5,
389 - temperature=0.0,
390 - extra_body=dict(
391 - # NOTE: this has to be true for n > 1 in vLLM, but not necessary
392 - # for official client.
393 - use_beam_search=True),
394 - )
395 - assert len(batch.choices) == 4
396 - assert batch.choices[0].text != batch.choices[
397 - 1].text, "beam search should be different"
398 - assert batch.choices[0].text == batch.choices[
399 - 2].text, "two copies of the same prompt should be the same"
400 - assert batch.choices[1].text == batch.choices[
401 - 3].text, "two copies of the same prompt should be the same"
402 -
403 - # test streaming
404 - batch = await client.completions.create(
405 - model=model_name,
406 - prompt=prompts,
407 - max_tokens=5,
408 - temperature=0.0,
409 - stream=True,
410 - )
411 - texts = [""] * 2
412 - async for chunk in batch:
413 - assert len(chunk.choices) == 1
414 - choice = chunk.choices[0]
415 - texts[choice.index] += choice.text
416 - assert texts[0] == texts[1]
417 -
418 -
419 -@pytest.mark.asyncio
420 -async def test_logits_bias(client: openai.AsyncOpenAI):
421 - prompt = "Hello, my name is"
422 - max_tokens = 5
423 - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
424 -
425 - # Test exclusive selection
426 - token_id = 1000
427 - completion = await client.completions.create(
428 - model=MODEL_NAME,
429 - prompt=prompt,
430 - max_tokens=max_tokens,
431 - temperature=0.0,
432 - logit_bias={str(token_id): 100},
433 - seed=42,
434 - )
435 - assert len(completion.choices[0].text) >= 5
436 - response_tokens = tokenizer(completion.choices[0].text,
437 - add_special_tokens=False)["input_ids"]
438 - expected_tokens = tokenizer(tokenizer.decode([token_id] * 5),
439 - add_special_tokens=False)["input_ids"]
440 - assert all([
441 - response == expected
442 - for response, expected in zip(response_tokens, expected_tokens)
443 - ])
444 -
445 - # Test ban
446 - completion = await client.completions.create(
447 - model=MODEL_NAME,
448 - prompt=prompt,
449 - max_tokens=max_tokens,
450 - temperature=0.0,
451 - )
452 - response_tokens = tokenizer(completion.choices[0].text,
453 - add_special_tokens=False)["input_ids"]
454 - first_response = completion.choices[0].text
455 - completion = await client.completions.create(
456 - model=MODEL_NAME,
457 - prompt=prompt,
458 - max_tokens=max_tokens,
459 - temperature=0.0,
460 - logit_bias={str(token): -100
461 - for token in response_tokens},
462 - )
463 - assert first_response != completion.choices[0].text
464 -
465 -
466 -@pytest.mark.asyncio
467 -@pytest.mark.parametrize("guided_decoding_backend",
468 - ["outlines", "lm-format-enforcer"])
469 -async def test_guided_json_completion(client: openai.AsyncOpenAI,
470 - guided_decoding_backend: str):
471 - completion = await client.completions.create(
472 - model=MODEL_NAME,
473 - prompt=f"Give an example JSON for an employee profile "
474 - f"that fits this schema: {TEST_SCHEMA}",
475 - n=3,
476 - temperature=1.0,
477 - max_tokens=500,
478 - extra_body=dict(guided_json=TEST_SCHEMA,
479 - guided_decoding_backend=guided_decoding_backend))
480 -
481 - assert completion.id is not None
482 - assert len(completion.choices) == 3
483 - for i in range(3):
484 - output_json = json.loads(completion.choices[i].text)
485 - jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
486 -
487 -
488 -@pytest.mark.asyncio
489 -@pytest.mark.parametrize("guided_decoding_backend",
490 - ["outlines", "lm-format-enforcer"])
491 -async def test_guided_regex_completion(client: openai.AsyncOpenAI,
492 - guided_decoding_backend: str):
493 - completion = await client.completions.create(
494 - model=MODEL_NAME,
495 - prompt=f"Give an example IPv4 address with this regex: {TEST_REGEX}",
496 - n=3,
497 - temperature=1.0,
498 - max_tokens=20,
499 - extra_body=dict(guided_regex=TEST_REGEX,
500 - guided_decoding_backend=guided_decoding_backend))
501 -
502 - assert completion.id is not None
503 - assert len(completion.choices) == 3
504 - for i in range(3):
505 - assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
506 -
507 -
508 -@pytest.mark.asyncio
509 -@pytest.mark.parametrize("guided_decoding_backend",
510 - ["outlines", "lm-format-enforcer"])
511 -async def test_guided_choice_completion(client: openai.AsyncOpenAI,
512 - guided_decoding_backend: str):
513 - completion = await client.completions.create(
514 - model=MODEL_NAME,
515 - prompt="The best language for type-safe systems programming is ",
516 - n=2,
517 - temperature=1.0,
518 - max_tokens=10,
519 - extra_body=dict(guided_choice=TEST_CHOICE,
520 - guided_decoding_backend=guided_decoding_backend))
521 -
522 - assert completion.id is not None
523 - assert len(completion.choices) == 2
524 - for i in range(2):
525 - assert completion.choices[i].text in TEST_CHOICE
526 -
527 -
528 -@pytest.mark.asyncio
529 -async def test_guided_grammar(client: openai.AsyncOpenAI):
530 - simple_sql_grammar = """
531 -start: select_statement
532 -
533 -select_statement: "SELECT" column "from" table "where" condition
534 -
535 -column: "col_1" | "col_2"
536 -table: "table_1" | "table_2"
537 -condition: column "=" number
538 -
539 -number: "1" | "2"
540 -"""
541 -
542 - completion = await client.completions.create(
543 - model=MODEL_NAME,
544 - prompt=("Generate a sql state that select col_1 from "
545 - "table_1 where it is equals to 1"),
546 - temperature=1.0,
547 - max_tokens=500,
548 - extra_body=dict(guided_grammar=simple_sql_grammar))
549 -
550 - content = completion.choices[0].text
551 -
552 - # use Lark to parse the output, and make sure it's a valid parse tree
553 - from lark import Lark
554 - parser = Lark(simple_sql_grammar)
555 - parser.parse(content)
556 -
557 - # remove spaces for comparison b/c we removed them in the grammar
558 - ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(" ", "")
559 -
560 - assert content.strip() == ground_truth
561 -
562 -
563 -@pytest.mark.asyncio
564 -@pytest.mark.parametrize(
565 - # first test base model, then test loras
566 - "model_name",
567 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
568 -)
569 -@pytest.mark.parametrize("logprobs_arg", [1, 0])
570 -async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
571 - model_name: str, logprobs_arg: int):
572 - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
573 - # test using text and token IDs
574 - for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
575 - completion = await client.completions.create(model=model_name,
576 - prompt=prompt,
577 - max_tokens=5,
578 - temperature=0.0,
579 - echo=True,
580 - logprobs=logprobs_arg)
581 -
582 - prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
583 - list) else prompt
584 - assert re.search(r"^" + prompt_text, completion.choices[0].text)
585 - logprobs = completion.choices[0].logprobs
586 - assert logprobs is not None
587 - assert len(logprobs.text_offset) > 5
588 - assert (len(logprobs.token_logprobs) > 5
589 - and logprobs.token_logprobs[0] is None)
590 - assert (len(logprobs.top_logprobs) > 5
591 - and logprobs.top_logprobs[0] is None)
592 - for top_logprobs in logprobs.top_logprobs[1:]:
593 - assert max(logprobs_arg,
594 - 1) <= len(top_logprobs) <= logprobs_arg + 1
595 - assert len(logprobs.tokens) > 5
596 -
597 -
598 -@pytest.mark.asyncio
599 -@pytest.mark.parametrize("guided_decoding_backend",
600 - ["outlines", "lm-format-enforcer"])
601 -async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
602 - guided_decoding_backend: str):
603 - with pytest.raises(openai.BadRequestError):
604 - _ = await client.completions.create(
605 - model=MODEL_NAME,
606 - prompt="Give an example JSON that fits this schema: 42",
607 - extra_body=dict(guided_json=42,
608 - guided_decoding_backend=guided_decoding_backend))
609 -
610 - with pytest.raises(openai.BadRequestError):
611 - _ = await client.completions.create(
612 - model=MODEL_NAME,
613 - prompt="Give an example string that fits this regex",
614 - extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
615 -
616 -
617 -@pytest.mark.asyncio
618 -@pytest.mark.parametrize(
619 - "model_name",
620 - [MODEL_NAME],
621 -)
622 -async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
623 - base_url = str(client.base_url)[:-3]
624 - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
625 -
626 - for add_special in [False, True]:
627 - prompt = "This is a test prompt."
628 - tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
629 -
630 - response = requests.post(base_url + "/tokenize",
631 - json={
632 - "add_special_tokens": add_special,
633 - "model": model_name,
634 - "prompt": prompt
635 - })
636 - response.raise_for_status()
637 - assert response.json() == {
638 - "tokens": tokens,
639 - "count": len(tokens),
640 - "max_model_len": 8192
641 - }
642 -
643 -
644 -@pytest.mark.asyncio
645 -@pytest.mark.parametrize(
646 - "model_name",
647 - [MODEL_NAME],
648 -)
649 -async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
650 - base_url = str(client.base_url)[:-3]
651 - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
652 -
653 - prompt = "This is a test prompt."
654 - tokens = tokenizer.encode(prompt, add_special_tokens=False)
655 -
656 - response = requests.post(base_url + "detokenize",
657 - json={
658 - "model": model_name,
659 - "tokens": tokens
660 - })
661 - response.raise_for_status()
662 - assert response.json() == {"prompt": prompt}
663 -
664 -
665 -if __name__ == "__main__":
666 - pytest.main([__file__])