@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b3104b2a10
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Fix logits processor when prompt_logprobs is not None (#3899)

胡译文 committed Apr 10, 2024 at 15:09 UTC b3104b2a10ab7cb7532442177ae0d0c40acf9d03
2 files changed +72 -1
tests/samplers/test_logits_processor.py new
+62
@@ -0,0 +1,62 @@
1 +import pytest
2 +import torch
3 +
4 +from vllm import SamplingParams
5 +
6 +MODELS = ["facebook/opt-125m"]
7 +
8 +
9 +@pytest.mark.parametrize("model", MODELS)
10 +@pytest.mark.parametrize("dtype", ["half"])
11 +def test_logits_processor_force_generate(
12 + vllm_runner,
13 + example_prompts,
14 + model: str,
15 + dtype: str,
16 +) -> None:
17 + vllm_model = vllm_runner(model, dtype=dtype)
18 + tokenizer = vllm_model.model.get_tokenizer()
19 + repeat_times = 2
20 + enforced_answers = " vLLM"
21 + vllm_token_ids = tokenizer.encode(enforced_answers,
22 + add_special_tokens=False)
23 + max_tokens = len(vllm_token_ids) * repeat_times
24 +
25 + def pick_vllm(token_ids, logits):
26 + token_id = vllm_token_ids[len(token_ids) % len(vllm_token_ids)]
27 + logits[token_id] = torch.finfo(logits.dtype).max
28 + return logits
29 +
30 + params_with_logprobs = SamplingParams(
31 + logits_processors=[pick_vllm],
32 + prompt_logprobs=3,
33 + max_tokens=max_tokens,
34 + )
35 +
36 + # test logits_processors when prompt_logprobs is not None
37 + vllm_model.model._add_request(
38 + prompt=example_prompts[0],
39 + sampling_params=params_with_logprobs,
40 + prompt_token_ids=None,
41 + )
42 +
43 + # test prompt_logprobs is not None
44 + vllm_model.model._add_request(
45 + prompt=example_prompts[1],
46 + sampling_params=SamplingParams(
47 + prompt_logprobs=3,
48 + max_tokens=max_tokens,
49 + ),
50 + prompt_token_ids=None,
51 + )
52 +
53 + # test grouped requests
54 + vllm_model.model._add_request(
55 + prompt=example_prompts[2],
56 + sampling_params=SamplingParams(max_tokens=max_tokens),
57 + prompt_token_ids=None,
58 + )
59 +
60 + outputs = vllm_model.model._run_engine(False)
61 +
62 + assert outputs[0].outputs[0].text == enforced_answers * repeat_times
vllm/model_executor/layers/logits_processor.py
+10 -1
@@ -86,8 +86,16 @@ def _apply_logits_processors(
86 ) -> torch.Tensor:
87 logits_row_idx = 0
88 found_logits_processors = False
89 - for seq_ids, sampling_params in sampling_metadata.seq_groups:
89 + for i, seq_group in enumerate(sampling_metadata.seq_groups):
90 + seq_ids, sampling_params = seq_group
91 logits_processors = sampling_params.logits_processors
92 + # handle prompt_logprobs by skipping rows in logits added for
93 + # the prompt tokens (prompt logprobs are not processed)
94 + if (i < sampling_metadata.num_prompts
95 + and sampling_params.prompt_logprobs is not None):
96 + assert len(seq_ids) == 1
97 + logits_row_idx += sampling_metadata.prompt_lens[i] - 1
98 +
99 if logits_processors:
100 found_logits_processors = True
101 for seq_id in seq_ids:
@@ -100,5 +108,6 @@ def _apply_logits_processors(
108 else:
109 logits_row_idx += len(seq_ids)
110 if found_logits_processors:
111 + # verifies that no rows in logits were missed unexpectedly
112 assert logits_row_idx == logits.shape[0]
113 return logits