@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0258b7a94b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] handle prompt_logprobs in _apply_min_tokens_penalty (#3876)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Apr 10, 2024 at 02:39 UTC 0258b7a94b08321ca01cf170f867b67c1920af87
2 files changed +112 -23
tests/samplers/test_sampler.py
+94 -22
@@ -1,3 +1,4 @@
1 +import itertools
2 import random
3 from typing import List, Optional, Tuple
4 from unittest.mock import patch
@@ -194,11 +195,15 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
195
196 def create_sampling_params(min_tokens,
197 eos_token_id=0,
197 - stop_token_ids=None):
198 + *,
199 + stop_token_ids: Optional[List[str]] = None,
200 + prompt_logprobs: Optional[int] = None):
201 sampling_params = SamplingParams(
202 min_tokens=min_tokens,
203 max_tokens=9999, # keep higher than max of min_tokens
204 stop_token_ids=stop_token_ids,
205 + # requesting prompt_logprobs changes the structure of `logits`
206 + prompt_logprobs=prompt_logprobs,
207 )
208 sampling_params.eos_token_id = eos_token_id
209 return sampling_params
@@ -217,9 +222,9 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
222
223 expected_penalization = []
224 sequence_metadata_list = []
225 + # 20% chance to generate seq group metadata list with all prompts
226 + is_prompt = random.random() < 0.2
227 while batch_size > 0:
221 - # 20% chance to generate prompt seq group with single sequence
222 - is_prompt = random.random() < 0.2
228 num_seqs = 1 if is_prompt else random.randint(1, batch_size)
229
230 eos_token_id = random.randint(0, VOCAB_SIZE - 1)
@@ -240,7 +245,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
245 seq_group_penalization = []
246 for _ in range(num_seqs):
247 num_input = random.randint(1, 100)
243 - num_generated = random.randint(1, 100) if not is_prompt else 0
248 + num_generated = 0 if is_prompt else random.randint(1, 100)
249 seq_data[next(seq_id_counter)] = create_sequence_data(
250 num_input=num_input, num_generated=num_generated)
251 seq_group_penalization.append(num_generated < min_tokens)
@@ -292,6 +297,21 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
297 ]
298 }
299
300 + prompt_with_penalization_and_prompt_logprobs = {
301 + "expected_penalization": [False, False, True],
302 + "seq_group_metadata_list": [
303 + SequenceGroupMetadata(
304 + request_id="test_1",
305 + is_prompt=True,
306 + seq_data={
307 + next(seq_id_counter): create_sequence_data(num_input=3),
308 + },
309 + sampling_params=create_sampling_params(1, prompt_logprobs=3),
310 + block_tables={},
311 + ),
312 + ]
313 + }
314 +
315 stop_penalizing_after_min_tokens = {
316 "expected_penalization": [False],
317 "seq_group_metadata_list": [
@@ -309,8 +329,34 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
329 }
330
331 stop_token_ids = [42, 99, 42, 0] # intentional duplication
312 - simple_combination = {
313 - "expected_penalization": [True, False, False],
332 + prompt_combination = {
333 + "expected_penalization": [False, True, False],
334 + "seq_group_metadata_list": [
335 + SequenceGroupMetadata(
336 + request_id="test_2",
337 + is_prompt=True,
338 + seq_data={
339 + next(seq_id_counter): create_sequence_data(num_input=2),
340 + },
341 + sampling_params=create_sampling_params(1, prompt_logprobs=3),
342 + block_tables={},
343 + ),
344 + SequenceGroupMetadata(
345 + request_id="test_3",
346 + is_prompt=True,
347 + seq_data={
348 + next(seq_id_counter): create_sequence_data(),
349 + },
350 + sampling_params=create_sampling_params(
351 + 0, stop_token_ids=stop_token_ids),
352 + block_tables={},
353 + )
354 + ]
355 + }
356 +
357 + stop_token_ids = [1, 999, 37, 37] # intentional duplication
358 + decode_combination = {
359 + "expected_penalization": [True, False, False, True, False],
360 "seq_group_metadata_list": [
361 SequenceGroupMetadata(
362 request_id="test_1",
@@ -327,14 +373,19 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
373 ),
374 SequenceGroupMetadata(
375 request_id="test_2",
330 - is_prompt=True,
376 + is_prompt=False,
377 seq_data={
332 - next(seq_id_counter): create_sequence_data(),
378 + next(seq_id_counter):
379 + create_sequence_data(num_generated=20),
380 + next(seq_id_counter):
381 + create_sequence_data(num_generated=1),
382 + next(seq_id_counter):
383 + create_sequence_data(num_generated=10),
384 },
385 sampling_params=create_sampling_params(
335 - 0, stop_token_ids=stop_token_ids),
386 + 10, prompt_logprobs=5, stop_token_ids=stop_token_ids),
387 block_tables={},
337 - )
388 + ),
389 ]
390 }
391
@@ -342,8 +393,10 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
393 test_cases = [
394 prompt_without_penalization,
395 prompt_with_penalization,
396 + prompt_with_penalization_and_prompt_logprobs,
397 stop_penalizing_after_min_tokens,
346 - simple_combination,
398 + prompt_combination,
399 + decode_combination,
400 ]
401 else:
402 test_cases = [generate_test_case()]
@@ -351,30 +404,49 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
404 def run_test_case(*,
405 expected_penalization=None,
406 seq_group_metadata_list=None):
354 - assert expected_penalization, "Invalid test case"
355 - assert seq_group_metadata_list, "Invalid test case"
407 + assert expected_penalization, \
408 + "Invalid test case, need expected_penalization"
409 + assert seq_group_metadata_list, \
410 + "Invalid test case, need seq_group_metadata_list"
411
412 batch_size = 0
413 prompt_lens = []
359 - sampling_params_per_seq = []
414 + sampling_params_per_row = []
415 for sgm in seq_group_metadata_list:
361 - num_seqs = len(sgm.seq_data)
362 - batch_size += num_seqs
416 sampling_params = sgm.sampling_params
364 - for seq_id in sgm.seq_data:
365 - prompt_lens.append(sgm.seq_data[seq_id].get_prompt_len())
366 - sampling_params_per_seq.append(sampling_params)
417 +
418 + num_rows = len(sgm.seq_data)
419 + if sgm.is_prompt:
420 + # a prompt seq_group has only one sequence
421 + seq_data = next(iter(sgm.seq_data.values()))
422 + prompt_len = seq_data.get_prompt_len()
423 + prompt_lens.append(prompt_len)
424 +
425 + if sgm.sampling_params.prompt_logprobs:
426 + # with prompt_logprobs each token in the prompt has a row in
427 + # logits
428 + num_rows = prompt_len
429 +
430 + batch_size += num_rows
431 + sampling_params_per_row.extend(
432 + itertools.repeat(sampling_params, num_rows))
433 +
434 + assert len(
435 + expected_penalization
436 + ) == batch_size, \
437 + ("Invalid test case, expected_penalization does not match computed"
438 + "batch size")
439
440 _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
441 sampling_metadata = model_runner._prepare_sample(
442 seq_group_metadata_list,
371 - prompt_lens=prompt_lens,
372 - subquery_lens=prompt_lens)
443 + prompt_lens=prompt_lens if prompt_lens else None,
444 + subquery_lens=prompt_lens if prompt_lens else None)
445 # the logits tensor is modified in-place by the sampler
446 _ = sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
447
448 for logits_idx, (should_penalize, sampling_params) in enumerate(
377 - zip(expected_penalization, sampling_params_per_seq)):
449 + zip(expected_penalization, sampling_params_per_row)):
450
451 tokens_to_check = [sampling_params.eos_token_id]
452 if sampling_params.stop_token_ids:
vllm/model_executor/layers/sampler.py
+18 -1
@@ -27,6 +27,12 @@ class Sampler(nn.Module):
27 6. Sample the next tokens.
28 Here, each sequence group within the batch can have different sampling
29 parameters (e.g., sampling method, temperature, top-p, top-k, etc.).
30 +
31 + The structure of the logits tensor is coupled with the seq_groups in
32 + sampling_metadata. Typically, each sequence in each seq_group has one row in
33 + logits for the next token to be sampled; however, for a seq_group with a
34 + prompt request with the prompt_logprobs sampling parameter, there are rows
35 + in logits for each token in the input prompt.
36 """
37
38 def forward(
@@ -106,7 +112,16 @@ def _apply_min_tokens_penalty(
112 # list of indices in logits that will be set to -inf
113 logits_to_penalize = []
114 start_idx = 0
109 - for seq_ids, sampling_params in sampling_metadata.seq_groups:
115 + for i, seq_group in enumerate(sampling_metadata.seq_groups):
116 + seq_ids, sampling_params = seq_group
117 +
118 + # handle prompt_logprobs by skipping rows in logits added for the prompt
119 + # tokens (prompt logprobs are not penalized)
120 + if (i < sampling_metadata.num_prompts
121 + and sampling_params.prompt_logprobs is not None):
122 + assert len(seq_ids) == 1
123 + start_idx += sampling_metadata.prompt_lens[i] - 1
124 +
125 min_tokens = sampling_params.min_tokens
126 if min_tokens > 0:
127 seqs_to_penalize = []
@@ -132,6 +147,8 @@ def _apply_min_tokens_penalty(
147 # eg. [ (1,2), (1,3), (5,6) ] -> ( (1,1,5), (2,3,6) )
148 logits[tuple(zip(*logits_to_penalize))] = -float("inf")
149
150 + # verifies that no rows in logits were missed unexpectedly
151 + assert start_idx == logits.shape[0]
152 return logits
153
154