A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Frontend] multiple sampling params support (#3570)
nunjunj committed
Apr 20, 2024 at 00:11 UTC
91528575ec0e6fd251bac08973a3abf23d4c318c
2 files changed
+61
-10
tests/entrypoints/test_llm_generate.py
new
+41
@@ -0,0 +1,41 @@
1
+import pytest
2
+
3
+from vllm import LLM, SamplingParams
4
+
5
+
6
+def test_multiple_sampling_params():
7
+
8
+ llm = LLM(model="facebook/opt-125m",
9
+ max_num_batched_tokens=4096,
10
+ tensor_parallel_size=1)
11
+
12
+ prompts = [
13
+ "Hello, my name is",
14
+ "The president of the United States is",
15
+ "The capital of France is",
16
+ "The future of AI is",
17
+ ]
18
+
19
+ sampling_params = [
20
+ SamplingParams(temperature=0.01, top_p=0.95),
21
+ SamplingParams(temperature=0.3, top_p=0.95),
22
+ SamplingParams(temperature=0.7, top_p=0.95),
23
+ SamplingParams(temperature=0.99, top_p=0.95),
24
+ ]
25
+
26
+ # Multiple SamplingParams should be matched with each prompt
27
+ outputs = llm.generate(prompts, sampling_params=sampling_params)
28
+ assert len(prompts) == len(outputs)
29
+
30
+ # Exception raised, if the size of params does not match the size of prompts
31
+ with pytest.raises(ValueError):
32
+ outputs = llm.generate(prompts, sampling_params=sampling_params[:3])
33
+
34
+ # Single SamplingParams should be applied to every prompt
35
+ single_sampling_params = SamplingParams(temperature=0.3, top_p=0.95)
36
+ outputs = llm.generate(prompts, sampling_params=single_sampling_params)
37
+ assert len(prompts) == len(outputs)
38
+
39
+ # sampling_params is None, default params should be applied
40
+ outputs = llm.generate(prompts, sampling_params=None)
41
+ assert len(prompts) == len(outputs)
\ No newline at end of file
vllm/entrypoints/llm.py
+20
-10
@@ -127,7 +127,8 @@ class LLM:
127
def generate(
128
self,
129
prompts: Optional[Union[str, List[str]]] = None,
130
- sampling_params: Optional[SamplingParams] = None,
130
+ sampling_params: Optional[Union[SamplingParams,
131
+ List[SamplingParams]]] = None,
132
prompt_token_ids: Optional[List[List[int]]] = None,
133
use_tqdm: bool = True,
134
lora_request: Optional[LoRARequest] = None,
@@ -142,7 +143,10 @@ class LLM:
143
Args:
144
prompts: A list of prompts to generate completions for.
145
sampling_params: The sampling parameters for text generation. If
145
- None, we use the default sampling parameters.
146
+ None, we use the default sampling parameters.
147
+ When it is a single value, it is applied to every prompt.
148
+ When it is a list, the list must have the same length as the
149
+ prompts and it is paired one by one with the prompt.
150
prompt_token_ids: A list of token IDs for the prompts. If None, we
151
use the tokenizer to convert the prompts to token IDs.
152
use_tqdm: Whether to use tqdm to display the progress bar.
@@ -163,27 +167,33 @@ class LLM:
167
and len(prompts) != len(prompt_token_ids)):
168
raise ValueError("The lengths of prompts and prompt_token_ids "
169
"must be the same.")
170
+
171
+ if prompts is not None:
172
+ num_requests = len(prompts)
173
+ else:
174
+ assert prompt_token_ids is not None
175
+ num_requests = len(prompt_token_ids)
176
+
177
if sampling_params is None:
178
# Use default sampling params.
179
sampling_params = SamplingParams()
180
181
+ elif isinstance(sampling_params,
182
+ list) and len(sampling_params) != num_requests:
183
+ raise ValueError("The lengths of prompts and sampling_params "
184
+ "must be the same.")
185
if multi_modal_data:
186
multi_modal_data.data = multi_modal_data.data.to(torch.float16)
187
188
# Add requests to the engine.
174
- if prompts is not None:
175
- num_requests = len(prompts)
176
- else:
177
- assert prompt_token_ids is not None
178
- num_requests = len(prompt_token_ids)
179
-
189
for i in range(num_requests):
190
prompt = prompts[i] if prompts is not None else None
191
token_ids = None if prompt_token_ids is None else prompt_token_ids[
192
i]
193
self._add_request(
194
prompt,
186
- sampling_params,
195
+ sampling_params[i]
196
+ if isinstance(sampling_params, list) else sampling_params,
197
token_ids,
198
lora_request=lora_request,
199
# Get ith image while maintaining the batch dim.
@@ -232,4 +242,4 @@ class LLM:
242
# This is necessary because some requests may be finished earlier than
243
# its previous requests.
244
outputs = sorted(outputs, key=lambda x: int(x.request_id))
235
- return outputs
245
+ return outputs
\ No newline at end of file