@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c13ad1b7bd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

feat: implement the min_tokens sampling parameter (#3124)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>

Travis Johnson committed Mar 25, 2024 at 11:14 UTC c13ad1b7bdf6fba5ab02225e6655bc26fae90192
5 files changed +299 -12
tests/samplers/test_sampler.py
+222 -2
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.sampler import Sampler
10 from vllm.model_executor.utils import set_random_seed
11 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12 from vllm.worker.model_runner import ModelRunner
13 +from vllm.utils import Counter
14
15
16 class MockLogitsSampler(Sampler):
@@ -25,9 +26,8 @@ class MockLogitsSampler(Sampler):
26 def _prepare_test(
27 batch_size: int
28 ) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, ModelRunner]:
28 - vocab_size = 32000
29 input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)
30 - fake_logits = torch.full((batch_size, vocab_size),
30 + fake_logits = torch.full((batch_size, VOCAB_SIZE),
31 1e-2,
32 dtype=input_tensor.dtype)
33 sampler = MockLogitsSampler(fake_logits)
@@ -35,6 +35,7 @@ def _prepare_test(
35 return input_tensor, fake_logits, sampler, model_runner
36
37
38 +VOCAB_SIZE = 32000
39 RANDOM_SEEDS = list(range(128))
40 CUDA_DEVICES = [
41 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
@@ -184,6 +185,225 @@ def test_sampler_all_beam(seed: int, device: str):
185 del model_runner
186
187
188 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
189 +@pytest.mark.parametrize("device", CUDA_DEVICES)
190 +def test_sampler_min_tokens_penalty(seed: int, device: str):
191 + seq_id_counter = Counter(start=random.randint(0, 100))
192 + set_random_seed(seed)
193 + torch.set_default_device(device)
194 +
195 + def create_sampling_params(min_tokens,
196 + eos_token_id=0,
197 + stop_token_ids=None):
198 + sampling_params = SamplingParams(
199 + min_tokens=min_tokens,
200 + max_tokens=9999, # keep higher than max of min_tokens
201 + stop_token_ids=stop_token_ids,
202 + )
203 + sampling_params.eos_token_id = eos_token_id
204 + return sampling_params
205 +
206 + def create_sequence_data(num_input=3, num_generated=0):
207 + seq_data = SequenceData(
208 + random.choices(range(0, VOCAB_SIZE), k=num_input))
209 + if num_generated > 0:
210 + seq_data.output_token_ids = random.choices(range(0, VOCAB_SIZE),
211 + k=num_generated)
212 + return seq_data
213 +
214 + def generate_test_case():
215 + # generate multiple seq groups but limit total batch size
216 + batch_size = random.randint(1, 128)
217 +
218 + expected_penalization = []
219 + sequence_metadata_list = []
220 + while batch_size > 0:
221 + # 20% chance to generate prompt seq group with single sequence
222 + is_prompt = random.random() < 0.2
223 + num_seqs = 1 if is_prompt else random.randint(1, batch_size)
224 +
225 + eos_token_id = random.randint(0, VOCAB_SIZE - 1)
226 + min_tokens = random.randint(0, 50)
227 + num_stop_tokens = random.randint(0, 8)
228 + if num_stop_tokens > 0:
229 + stop_token_ids = random.choices(range(0, VOCAB_SIZE - 1),
230 + k=num_stop_tokens)
231 + else:
232 + stop_token_ids = None
233 +
234 + sampling_params = create_sampling_params(
235 + min_tokens=min_tokens,
236 + eos_token_id=eos_token_id,
237 + stop_token_ids=stop_token_ids)
238 +
239 + seq_data = {}
240 + seq_group_penalization = []
241 + for _ in range(num_seqs):
242 + num_input = random.randint(1, 100)
243 + num_generated = random.randint(1, 100) if not is_prompt else 0
244 + seq_data[next(seq_id_counter)] = create_sequence_data(
245 + num_input=num_input, num_generated=num_generated)
246 + seq_group_penalization.append(num_generated < min_tokens)
247 +
248 + expected_penalization.extend(seq_group_penalization)
249 + sequence_metadata_list.append(
250 + SequenceGroupMetadata(
251 + request_id=f"test_{batch_size}",
252 + is_prompt=is_prompt,
253 + seq_data=seq_data,
254 + sampling_params=sampling_params,
255 + block_tables={},
256 + ))
257 + batch_size -= num_seqs
258 +
259 + return {
260 + "expected_penalization": expected_penalization,
261 + "seq_group_metadata_list": sequence_metadata_list,
262 + }
263 +
264 + # define some explicit test cases for edge case behavior
265 + prompt_without_penalization = {
266 + "expected_penalization": [False],
267 + "seq_group_metadata_list": [
268 + SequenceGroupMetadata(
269 + request_id="test_1",
270 + is_prompt=True,
271 + seq_data={
272 + next(seq_id_counter): create_sequence_data(),
273 + },
274 + sampling_params=create_sampling_params(0),
275 + block_tables={},
276 + ),
277 + ]
278 + }
279 +
280 + prompt_with_penalization = {
281 + "expected_penalization": [True],
282 + "seq_group_metadata_list": [
283 + SequenceGroupMetadata(
284 + request_id="test_1",
285 + is_prompt=True,
286 + seq_data={
287 + next(seq_id_counter): create_sequence_data(),
288 + },
289 + sampling_params=create_sampling_params(1),
290 + block_tables={},
291 + ),
292 + ]
293 + }
294 +
295 + stop_penalizing_after_min_tokens = {
296 + "expected_penalization": [False],
297 + "seq_group_metadata_list": [
298 + SequenceGroupMetadata(
299 + request_id="test_1",
300 + is_prompt=False,
301 + seq_data={
302 + next(seq_id_counter):
303 + create_sequence_data(num_generated=1),
304 + },
305 + sampling_params=create_sampling_params(1),
306 + block_tables={},
307 + )
308 + ]
309 + }
310 +
311 + stop_token_ids = [42, 99, 42, 0] # intentional duplication
312 + simple_combination = {
313 + "expected_penalization": [True, False, False],
314 + "seq_group_metadata_list": [
315 + SequenceGroupMetadata(
316 + request_id="test_1",
317 + is_prompt=False,
318 + seq_data={
319 + next(seq_id_counter):
320 + create_sequence_data(num_generated=1),
321 + next(seq_id_counter):
322 + create_sequence_data(num_generated=100),
323 + },
324 + sampling_params=create_sampling_params(
325 + 2, stop_token_ids=stop_token_ids),
326 + block_tables={},
327 + ),
328 + SequenceGroupMetadata(
329 + request_id="test_2",
330 + is_prompt=True,
331 + seq_data={
332 + next(seq_id_counter): create_sequence_data(),
333 + },
334 + sampling_params=create_sampling_params(
335 + 0, stop_token_ids=stop_token_ids),
336 + block_tables={},
337 + )
338 + ]
339 + }
340 +
341 + if seed == 0:
342 + test_cases = [
343 + prompt_without_penalization,
344 + prompt_with_penalization,
345 + stop_penalizing_after_min_tokens,
346 + simple_combination,
347 + ]
348 + else:
349 + test_cases = [generate_test_case()]
350 +
351 + def run_test_case(*,
352 + expected_penalization=None,
353 + seq_group_metadata_list=None):
354 + assert expected_penalization, "Invalid test case"
355 + assert seq_group_metadata_list, "Invalid test case"
356 +
357 + batch_size = 0
358 + prompt_lens = []
359 + sampling_params_per_seq = []
360 + for sgm in seq_group_metadata_list:
361 + num_seqs = len(sgm.seq_data)
362 + batch_size += num_seqs
363 + sampling_params = sgm.sampling_params
364 + for seq_id in sgm.seq_data:
365 + prompt_lens.append(sgm.seq_data[seq_id].get_prompt_len())
366 + sampling_params_per_seq.append(sampling_params)
367 +
368 + _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
369 + sampling_metadata = model_runner._prepare_sample(
370 + seq_group_metadata_list,
371 + prompt_lens=prompt_lens,
372 + subquery_lens=prompt_lens)
373 + # the logits tensor is modified in-place by the sampler
374 + _ = sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
375 +
376 + for logits_idx, (should_penalize, sampling_params) in enumerate(
377 + zip(expected_penalization, sampling_params_per_seq)):
378 +
379 + tokens_to_check = [sampling_params.eos_token_id]
380 + if sampling_params.stop_token_ids:
381 + tokens_to_check.extend(sampling_params.stop_token_ids)
382 + tokens_to_check = set(tokens_to_check)
383 +
384 + if should_penalize:
385 + for token_id in tokens_to_check:
386 + assert fake_logits[logits_idx, token_id] == -float(
387 + 'inf'
388 + ), f"Expected token {token_id} for logits row {logits_idx}"
389 + " to be penalized"
390 + # no other tokens should be set to -inf
391 + assert torch.count_nonzero(
392 + fake_logits[logits_idx, :] == -float('inf')) == len(
393 + tokens_to_check
394 + ), f"Expected only {len(tokens_to_check)} to be penalized"
395 + else:
396 + # no tokens should be set to -inf
397 + assert torch.count_nonzero(
398 + fake_logits[logits_idx, :] ==
399 + -float('inf')) == 0, "No tokens should have been penalized"
400 +
401 + del model_runner
402 +
403 + for test_case in test_cases:
404 + run_test_case(**test_case)
405 +
406 +
407 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
408 @pytest.mark.parametrize("device", CUDA_DEVICES)
409 def test_sampler_mixed(seed: int, device: str):
vllm/engine/llm_engine.py
+18 -10
@@ -282,6 +282,9 @@ class LLMEngine:
282 # Defensive copy of SamplingParams, which are used by the sampler,
283 # this doesn't deep-copy LogitsProcessor objects
284 sampling_params = sampling_params.clone()
285 + # inject the eos token id into the sampling_params to support min_tokens
286 + # processing
287 + sampling_params.eos_token_id = seq.eos_token_id
288
289 # Create the sequence group.
290 seq_group = SequenceGroup(request_id, [seq], sampling_params,
@@ -713,6 +716,21 @@ class LLMEngine:
716 def _check_stop(self, seq: Sequence,
717 sampling_params: SamplingParams) -> None:
718 """Stop the finished sequences."""
719 + # Check if the sequence has reached max_model_len.
720 + if seq.get_len() > self.scheduler_config.max_model_len:
721 + seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
722 + return
723 +
724 + # Check if the sequence has reached max_tokens.
725 + if seq.get_output_len() == sampling_params.max_tokens:
726 + seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
727 + return
728 +
729 + # Check if the minimum number of tokens has been generated yet;
730 + # skip the stop string/token checks if not
731 + if seq.get_output_len() < sampling_params.min_tokens:
732 + return
733 +
734 for stop_str in sampling_params.stop:
735 if seq.output_text.endswith(stop_str):
736 self._finalize_sequence(seq, sampling_params, stop_str)
@@ -725,16 +743,6 @@ class LLMEngine:
743 seq.status = SequenceStatus.FINISHED_STOPPED
744 return
745
728 - # Check if the sequence has reached max_model_len.
729 - if seq.get_len() > self.scheduler_config.max_model_len:
730 - seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
731 - return
732 -
733 - # Check if the sequence has reached max_tokens.
734 - if seq.get_output_len() == sampling_params.max_tokens:
735 - seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
736 - return
737 -
746 # Check if the sequence has generated the EOS token.
747 if ((not sampling_params.ignore_eos)
748 and seq.get_last_token_id() == seq.eos_token_id):
vllm/entrypoints/openai/protocol.py
+4
@@ -88,6 +88,7 @@ class ChatCompletionRequest(BaseModel):
88 length_penalty: Optional[float] = 1.0
89 early_stopping: Optional[bool] = False
90 ignore_eos: Optional[bool] = False
91 + min_tokens: Optional[int] = 0
92 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
93 skip_special_tokens: Optional[bool] = True
94 spaces_between_special_tokens: Optional[bool] = True
@@ -165,6 +166,7 @@ class ChatCompletionRequest(BaseModel):
166 stop=self.stop,
167 stop_token_ids=self.stop_token_ids,
168 max_tokens=self.max_tokens,
169 + min_tokens=self.min_tokens,
170 logprobs=self.top_logprobs if self.logprobs else None,
171 prompt_logprobs=self.top_logprobs if self.echo else None,
172 best_of=self.best_of,
@@ -224,6 +226,7 @@ class CompletionRequest(BaseModel):
226 early_stopping: Optional[bool] = False
227 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
228 ignore_eos: Optional[bool] = False
229 + min_tokens: Optional[int] = 0
230 skip_special_tokens: Optional[bool] = True
231 spaces_between_special_tokens: Optional[bool] = True
232 # doc: end-completion-sampling-params
@@ -296,6 +299,7 @@ class CompletionRequest(BaseModel):
299 stop_token_ids=self.stop_token_ids,
300 ignore_eos=self.ignore_eos,
301 max_tokens=self.max_tokens if not echo_without_generation else 1,
302 + min_tokens=self.min_tokens,
303 logprobs=self.logprobs,
304 use_beam_search=self.use_beam_search,
305 early_stopping=self.early_stopping,
vllm/model_executor/layers/sampler.py
+41
@@ -1,4 +1,5 @@
1 """A layer that samples the next tokens from the model's outputs."""
2 +import itertools
3 from typing import Dict, List, Optional, Tuple
4
5 import torch
@@ -36,6 +37,10 @@ class Sampler(nn.Module):
37 assert logits is not None
38 _, vocab_size = logits.shape
39
40 + # Apply min_tokens penalty which sets stop tokens to -inf if min_tokens
41 + # have not been generated yet
42 + logits = _apply_min_tokens_penalty(logits, sampling_metadata)
43 +
44 # Prepare sampling tensors with pinned memory to avoid blocking.
45 (sampling_tensors, do_penalties, do_top_p_top_k,
46 do_min_p) = SamplingTensors.from_sampling_metadata(
@@ -94,6 +99,42 @@ def _get_bin_counts_and_mask(
99 return bin_counts, mask
100
101
102 +def _apply_min_tokens_penalty(
103 + logits: torch.Tensor,
104 + sampling_metadata: SamplingMetadata,
105 +) -> torch.Tensor:
106 + # list of indices in logits that will be set to -inf
107 + logits_to_penalize = []
108 + start_idx = 0
109 + for seq_ids, sampling_params in sampling_metadata.seq_groups:
110 + min_tokens = sampling_params.min_tokens
111 + if min_tokens > 0:
112 + seqs_to_penalize = []
113 + for i, seq_id in enumerate(seq_ids):
114 + seq_data = sampling_metadata.seq_data[seq_id]
115 + if len(seq_data.output_token_ids) < min_tokens:
116 + seqs_to_penalize.append(i)
117 +
118 + if seqs_to_penalize:
119 + # convert to the index into logits
120 + seqs_to_penalize = [start_idx + i for i in seqs_to_penalize]
121 + # use set() to remove any duplicates
122 + token_ids_to_penalize = set(sampling_params.stop_token_ids +
123 + [sampling_params.eos_token_id])
124 + # itertools.product pairs each seq index with every token id
125 + logits_to_penalize.extend(
126 + itertools.product(seqs_to_penalize, token_ids_to_penalize))
127 +
128 + start_idx += len(seq_ids)
129 +
130 + if logits_to_penalize:
131 + # use zip and * to group indices along each dimension
132 + # eg. [ (1,2), (1,3), (5,6) ] -> ( (1,1,5), (2,3,6) )
133 + logits[tuple(zip(*logits_to_penalize))] = -float("inf")
134 +
135 + return logits
136 +
137 +
138 def _apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
139 output_tokens_tensor: torch.Tensor,
140 presence_penalties: torch.Tensor,
vllm/sampling_params.py
+14
@@ -79,6 +79,8 @@ class SamplingParams:
79 ignore_eos: Whether to ignore the EOS token and continue generating
80 tokens after the EOS token is generated.
81 max_tokens: Maximum number of tokens to generate per output sequence.
82 + min_tokens: Minimum number of tokens to generate per output sequence
83 + before EOS or stop_token_ids can be generated
84 logprobs: Number of log probabilities to return per output token.
85 Note that the implementation follows the OpenAI API: The return
86 result includes the log probabilities on the `logprobs` most likely
@@ -113,6 +115,7 @@ class SamplingParams:
115 include_stop_str_in_output: bool = False,
116 ignore_eos: bool = False,
117 max_tokens: Optional[int] = 16,
118 + min_tokens: int = 0,
119 logprobs: Optional[int] = None,
120 prompt_logprobs: Optional[int] = None,
121 skip_special_tokens: bool = True,
@@ -144,6 +147,7 @@ class SamplingParams:
147 self.stop_token_ids = list(stop_token_ids)
148 self.ignore_eos = ignore_eos
149 self.max_tokens = max_tokens
150 + self.min_tokens = min_tokens
151 self.logprobs = logprobs
152 self.prompt_logprobs = prompt_logprobs
153 self.skip_special_tokens = skip_special_tokens
@@ -161,6 +165,8 @@ class SamplingParams:
165 self.top_k = -1
166 self.min_p = 0.0
167 self._verify_greedy_sampling()
168 + # injected by the engine
169 + self.eos_token_id = None
170
171 def _verify_args(self) -> None:
172 if self.n < 1:
@@ -191,6 +197,13 @@ class SamplingParams:
197 if self.max_tokens is not None and self.max_tokens < 1:
198 raise ValueError(
199 f"max_tokens must be at least 1, got {self.max_tokens}.")
200 + if self.min_tokens < 0:
201 + raise ValueError(f"min_tokens must be greater than or equal to 0, "
202 + f"got {self.min_tokens}.")
203 + if self.max_tokens is not None and self.min_tokens > self.max_tokens:
204 + raise ValueError(
205 + f"min_tokens must be less than or equal to "
206 + f"max_tokens={self.max_tokens}, got {self.min_tokens}.")
207 if self.logprobs is not None and self.logprobs < 0:
208 raise ValueError(
209 f"logprobs must be non-negative, got {self.logprobs}.")
@@ -272,6 +285,7 @@ class SamplingParams:
285 f"include_stop_str_in_output={self.include_stop_str_in_output}, "
286 f"ignore_eos={self.ignore_eos}, "
287 f"max_tokens={self.max_tokens}, "
288 + f"min_tokens={self.min_tokens}, "
289 f"logprobs={self.logprobs}, "
290 f"prompt_logprobs={self.prompt_logprobs}, "
291 f"skip_special_tokens={self.skip_special_tokens}, "