@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 91528575ec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Frontend] multiple sampling params support (#3570)

nunjunj committed Apr 20, 2024 at 00:11 UTC 91528575ec0e6fd251bac08973a3abf23d4c318c
2 files changed +61 -10
tests/entrypoints/test_llm_generate.py new
+41
@@ -0,0 +1,41 @@
1 +import pytest
2 +
3 +from vllm import LLM, SamplingParams
4 +
5 +
6 +def test_multiple_sampling_params():
7 +
8 + llm = LLM(model="facebook/opt-125m",
9 + max_num_batched_tokens=4096,
10 + tensor_parallel_size=1)
11 +
12 + prompts = [
13 + "Hello, my name is",
14 + "The president of the United States is",
15 + "The capital of France is",
16 + "The future of AI is",
17 + ]
18 +
19 + sampling_params = [
20 + SamplingParams(temperature=0.01, top_p=0.95),
21 + SamplingParams(temperature=0.3, top_p=0.95),
22 + SamplingParams(temperature=0.7, top_p=0.95),
23 + SamplingParams(temperature=0.99, top_p=0.95),
24 + ]
25 +
26 + # Multiple SamplingParams should be matched with each prompt
27 + outputs = llm.generate(prompts, sampling_params=sampling_params)
28 + assert len(prompts) == len(outputs)
29 +
30 + # Exception raised, if the size of params does not match the size of prompts
31 + with pytest.raises(ValueError):
32 + outputs = llm.generate(prompts, sampling_params=sampling_params[:3])
33 +
34 + # Single SamplingParams should be applied to every prompt
35 + single_sampling_params = SamplingParams(temperature=0.3, top_p=0.95)
36 + outputs = llm.generate(prompts, sampling_params=single_sampling_params)
37 + assert len(prompts) == len(outputs)
38 +
39 + # sampling_params is None, default params should be applied
40 + outputs = llm.generate(prompts, sampling_params=None)
41 + assert len(prompts) == len(outputs)
\ No newline at end of file
vllm/entrypoints/llm.py
+20 -10
@@ -127,7 +127,8 @@ class LLM:
127 def generate(
128 self,
129 prompts: Optional[Union[str, List[str]]] = None,
130 - sampling_params: Optional[SamplingParams] = None,
130 + sampling_params: Optional[Union[SamplingParams,
131 + List[SamplingParams]]] = None,
132 prompt_token_ids: Optional[List[List[int]]] = None,
133 use_tqdm: bool = True,
134 lora_request: Optional[LoRARequest] = None,
@@ -142,7 +143,10 @@ class LLM:
143 Args:
144 prompts: A list of prompts to generate completions for.
145 sampling_params: The sampling parameters for text generation. If
145 - None, we use the default sampling parameters.
146 + None, we use the default sampling parameters.
147 + When it is a single value, it is applied to every prompt.
148 + When it is a list, the list must have the same length as the
149 + prompts and it is paired one by one with the prompt.
150 prompt_token_ids: A list of token IDs for the prompts. If None, we
151 use the tokenizer to convert the prompts to token IDs.
152 use_tqdm: Whether to use tqdm to display the progress bar.
@@ -163,27 +167,33 @@ class LLM:
167 and len(prompts) != len(prompt_token_ids)):
168 raise ValueError("The lengths of prompts and prompt_token_ids "
169 "must be the same.")
170 +
171 + if prompts is not None:
172 + num_requests = len(prompts)
173 + else:
174 + assert prompt_token_ids is not None
175 + num_requests = len(prompt_token_ids)
176 +
177 if sampling_params is None:
178 # Use default sampling params.
179 sampling_params = SamplingParams()
180
181 + elif isinstance(sampling_params,
182 + list) and len(sampling_params) != num_requests:
183 + raise ValueError("The lengths of prompts and sampling_params "
184 + "must be the same.")
185 if multi_modal_data:
186 multi_modal_data.data = multi_modal_data.data.to(torch.float16)
187
188 # Add requests to the engine.
174 - if prompts is not None:
175 - num_requests = len(prompts)
176 - else:
177 - assert prompt_token_ids is not None
178 - num_requests = len(prompt_token_ids)
179 -
189 for i in range(num_requests):
190 prompt = prompts[i] if prompts is not None else None
191 token_ids = None if prompt_token_ids is None else prompt_token_ids[
192 i]
193 self._add_request(
194 prompt,
186 - sampling_params,
195 + sampling_params[i]
196 + if isinstance(sampling_params, list) else sampling_params,
197 token_ids,
198 lora_request=lora_request,
199 # Get ith image while maintaining the batch dim.
@@ -232,4 +242,4 @@ class LLM:
242 # This is necessary because some requests may be finished earlier than
243 # its previous requests.
244 outputs = sorted(outputs, key=lambda x: int(x.request_id))
235 - return outputs
245 + return outputs
\ No newline at end of file