@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d92b3c5cde
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix][CI/Build] Test prompt adapters in openai entrypoint tests (#6419)

Joe committed Jul 15, 2024 at 18:54 UTC d92b3c5cdef59533347ac714a70274f186943019
2 files changed +54 -31
tests/entrypoints/openai/test_completion.py
+51 -29
@@ -17,9 +17,13 @@ from ...utils import RemoteOpenAIServer
17
18 # any model with a chat template should work here
19 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
20 -# technically this needs Mistral-7B-v0.1 as base, but we're not testing
21 -# generation quality here
20 +# technically these adapters use a different base model,
21 +# but we're not testing generation quality here
22 LORA_NAME = "typeof/zephyr-7b-beta-lora"
23 +PA_NAME = "swapnilbp/llama_tweet_ptune"
24 +# if PA_NAME changes, PA_NUM_VIRTUAL_TOKENS might also
25 +# need to change to match the prompt adapter
26 +PA_NUM_VIRTUAL_TOKENS = 8
27
28
29 @pytest.fixture(scope="module")
@@ -28,7 +32,12 @@ def zephyr_lora_files():
32
33
34 @pytest.fixture(scope="module")
31 -def server(zephyr_lora_files):
35 +def zephyr_pa_files():
36 + return snapshot_download(repo_id=PA_NAME)
37 +
38 +
39 +@pytest.fixture(scope="module")
40 +def server(zephyr_lora_files, zephyr_pa_files):
41 with RemoteOpenAIServer([
42 "--model",
43 MODEL_NAME,
@@ -37,8 +46,10 @@ def server(zephyr_lora_files):
46 "bfloat16",
47 "--max-model-len",
48 "8192",
49 + "--max-num-seqs",
50 + "128",
51 "--enforce-eager",
41 - # lora config below
52 + # lora config
53 "--enable-lora",
54 "--lora-modules",
55 f"zephyr-lora={zephyr_lora_files}",
@@ -47,7 +58,14 @@ def server(zephyr_lora_files):
58 "64",
59 "--max-cpu-loras",
60 "2",
50 - "--max-num-seqs",
61 + # pa config
62 + "--enable-prompt-adapter",
63 + "--prompt-adapters",
64 + f"zephyr-pa={zephyr_pa_files}",
65 + f"zephyr-pa2={zephyr_pa_files}",
66 + "--max-prompt-adapters",
67 + "2",
68 + "--max-prompt-adapter-token",
69 "128",
70 ]) as remote_server:
71 yield remote_server
@@ -60,11 +78,14 @@ def client(server):
78
79 @pytest.mark.asyncio
80 @pytest.mark.parametrize(
63 - # first test base model, then test loras
64 - "model_name",
65 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
81 + # first test base model, then test loras, then test prompt adapters
82 + "model_name,num_virtual_tokens",
83 + [(MODEL_NAME, 0), ("zephyr-lora", 0), ("zephyr-lora2", 0),
84 + ("zephyr-pa", PA_NUM_VIRTUAL_TOKENS),
85 + ("zephyr-pa2", PA_NUM_VIRTUAL_TOKENS)],
86 )
67 -async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
87 +async def test_single_completion(client: openai.AsyncOpenAI, model_name: str,
88 + num_virtual_tokens: int):
89 completion = await client.completions.create(model=model_name,
90 prompt="Hello, my name is",
91 max_tokens=5,
@@ -77,28 +98,30 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
98 assert len(choice.text) >= 5
99 assert choice.finish_reason == "length"
100 assert completion.usage == openai.types.CompletionUsage(
80 - completion_tokens=5, prompt_tokens=6, total_tokens=11)
101 + completion_tokens=5,
102 + prompt_tokens=6 + num_virtual_tokens,
103 + total_tokens=11 + num_virtual_tokens)
104
105 # test using token IDs
106 completion = await client.completions.create(
84 - model=MODEL_NAME,
107 + model=model_name,
108 prompt=[0, 0, 0, 0, 0],
109 max_tokens=5,
110 temperature=0.0,
111 )
89 - assert len(completion.choices[0].text) >= 5
112 + assert len(completion.choices[0].text) >= 1
113
114
115 @pytest.mark.asyncio
116 @pytest.mark.parametrize(
94 - # first test base model, then test loras
117 + # first test base model, then test loras, then test prompt adapters
118 "model_name",
96 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
119 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2", "zephyr-pa", "zephyr-pa2"],
120 )
121 async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
122 # test using token IDs
123 completion = await client.completions.create(
101 - model=MODEL_NAME,
124 + model=model_name,
125 prompt=[0, 0, 0, 0, 0],
126 max_tokens=5,
127 temperature=0.0,
@@ -110,14 +133,14 @@ async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
133
134 @pytest.mark.asyncio
135 @pytest.mark.parametrize(
113 - # just test 1 lora hereafter
136 + # just test 1 lora and 1 pa hereafter
137 "model_name",
115 - [MODEL_NAME, "zephyr-lora"],
138 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
139 )
140 async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
141 # test using token IDs
142 completion = await client.completions.create(
120 - model=MODEL_NAME,
143 + model=model_name,
144 prompt=[0, 0, 0, 0, 0],
145 max_tokens=5,
146 temperature=0.0,
@@ -133,12 +156,12 @@ async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
156 @pytest.mark.asyncio
157 @pytest.mark.parametrize(
158 "model_name",
136 - [MODEL_NAME, "zephyr-lora"],
159 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
160 )
161 async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
162 # test using token IDs
163 completion = await client.completions.create(
141 - model=MODEL_NAME,
164 + model=model_name,
165 prompt=[0, 0, 0, 0, 0],
166 max_tokens=5,
167 temperature=0.0,
@@ -154,7 +177,7 @@ async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
177 @pytest.mark.asyncio
178 @pytest.mark.parametrize(
179 "model_name",
157 - [MODEL_NAME, "zephyr-lora"],
180 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
181 )
182 async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
183 model_name: str):
@@ -162,7 +185,7 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
185 with pytest.raises(
186 (openai.BadRequestError, openai.APIError)): # test using token IDs
187 await client.completions.create(
165 - model=MODEL_NAME,
188 + model=model_name,
189 prompt=[0, 0, 0, 0, 0],
190 max_tokens=5,
191 temperature=0.0,
@@ -174,7 +197,7 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
197 with pytest.raises(
198 (openai.BadRequestError, openai.APIError)): # test using token IDs
199 stream = await client.completions.create(
177 - model=MODEL_NAME,
200 + model=model_name,
201 prompt=[0, 0, 0, 0, 0],
202 max_tokens=5,
203 temperature=0.0,
@@ -199,7 +222,7 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
222 @pytest.mark.asyncio
223 @pytest.mark.parametrize(
224 "model_name",
202 - [MODEL_NAME, "zephyr-lora"],
225 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
226 )
227 async def test_completion_streaming(client: openai.AsyncOpenAI,
228 model_name: str):
@@ -233,7 +256,7 @@ async def test_completion_streaming(client: openai.AsyncOpenAI,
256 @pytest.mark.asyncio
257 @pytest.mark.parametrize(
258 "model_name",
236 - ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
259 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
260 )
261 async def test_completion_stream_options(client: openai.AsyncOpenAI,
262 model_name: str):
@@ -369,9 +392,8 @@ async def test_completion_stream_options(client: openai.AsyncOpenAI,
392
393 @pytest.mark.asyncio
394 @pytest.mark.parametrize(
372 - # just test 1 lora hereafter
395 "model_name",
374 - [MODEL_NAME, "zephyr-lora"],
396 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
397 )
398 async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
399 # test both text and token IDs
@@ -623,7 +645,7 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
645 )
646 async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
647 base_url = str(client.base_url)[:-3].strip("/")
626 - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
648 + tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
649
650 for add_special in [False, True]:
651 prompt = "This is a test prompt."
@@ -650,7 +672,7 @@ async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
672 )
673 async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
674 base_url = str(client.base_url)[:-3]
653 - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
675 + tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
676
677 prompt = "This is a test prompt."
678 tokens = tokenizer.encode(prompt, add_special_tokens=False)
vllm/entrypoints/openai/serving_engine.py
+3 -2
@@ -1,4 +1,5 @@
1 import json
2 +import pathlib
3 from dataclasses import dataclass
4 from http import HTTPStatus
5 from typing import Any, Dict, List, Optional, Tuple, Union
@@ -74,8 +75,8 @@ class OpenAIServing:
75 self.prompt_adapter_requests = []
76 if prompt_adapters is not None:
77 for i, prompt_adapter in enumerate(prompt_adapters, start=1):
77 - with open(f"./{prompt_adapter.local_path}"
78 - f"/adapter_config.json") as f:
78 + with pathlib.Path(prompt_adapter.local_path,
79 + "adapter_config.json").open() as f:
80 adapter_config = json.load(f)
81 num_virtual_tokens = adapter_config["num_virtual_tokens"]
82 self.prompt_adapter_requests.append(