A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Fix logits processor when prompt_logprobs is not None (#3899)
胡译文 committed
Apr 10, 2024 at 15:09 UTC
b3104b2a10ab7cb7532442177ae0d0c40acf9d03
2 files changed
+72
-1
tests/samplers/test_logits_processor.py
new
+62
@@ -0,0 +1,62 @@
1
+import pytest
2
+import torch
3
+
4
+from vllm import SamplingParams
5
+
6
+MODELS = ["facebook/opt-125m"]
7
+
8
+
9
+@pytest.mark.parametrize("model", MODELS)
10
+@pytest.mark.parametrize("dtype", ["half"])
11
+def test_logits_processor_force_generate(
12
+ vllm_runner,
13
+ example_prompts,
14
+ model: str,
15
+ dtype: str,
16
+) -> None:
17
+ vllm_model = vllm_runner(model, dtype=dtype)
18
+ tokenizer = vllm_model.model.get_tokenizer()
19
+ repeat_times = 2
20
+ enforced_answers = " vLLM"
21
+ vllm_token_ids = tokenizer.encode(enforced_answers,
22
+ add_special_tokens=False)
23
+ max_tokens = len(vllm_token_ids) * repeat_times
24
+
25
+ def pick_vllm(token_ids, logits):
26
+ token_id = vllm_token_ids[len(token_ids) % len(vllm_token_ids)]
27
+ logits[token_id] = torch.finfo(logits.dtype).max
28
+ return logits
29
+
30
+ params_with_logprobs = SamplingParams(
31
+ logits_processors=[pick_vllm],
32
+ prompt_logprobs=3,
33
+ max_tokens=max_tokens,
34
+ )
35
+
36
+ # test logits_processors when prompt_logprobs is not None
37
+ vllm_model.model._add_request(
38
+ prompt=example_prompts[0],
39
+ sampling_params=params_with_logprobs,
40
+ prompt_token_ids=None,
41
+ )
42
+
43
+ # test prompt_logprobs is not None
44
+ vllm_model.model._add_request(
45
+ prompt=example_prompts[1],
46
+ sampling_params=SamplingParams(
47
+ prompt_logprobs=3,
48
+ max_tokens=max_tokens,
49
+ ),
50
+ prompt_token_ids=None,
51
+ )
52
+
53
+ # test grouped requests
54
+ vllm_model.model._add_request(
55
+ prompt=example_prompts[2],
56
+ sampling_params=SamplingParams(max_tokens=max_tokens),
57
+ prompt_token_ids=None,
58
+ )
59
+
60
+ outputs = vllm_model.model._run_engine(False)
61
+
62
+ assert outputs[0].outputs[0].text == enforced_answers * repeat_times
vllm/model_executor/layers/logits_processor.py
+10
-1
@@ -86,8 +86,16 @@ def _apply_logits_processors(
86
) -> torch.Tensor:
87
logits_row_idx = 0
88
found_logits_processors = False
89
- for seq_ids, sampling_params in sampling_metadata.seq_groups:
89
+ for i, seq_group in enumerate(sampling_metadata.seq_groups):
90
+ seq_ids, sampling_params = seq_group
91
logits_processors = sampling_params.logits_processors
92
+ # handle prompt_logprobs by skipping rows in logits added for
93
+ # the prompt tokens (prompt logprobs are not processed)
94
+ if (i < sampling_metadata.num_prompts
95
+ and sampling_params.prompt_logprobs is not None):
96
+ assert len(seq_ids) == 1
97
+ logits_row_idx += sampling_metadata.prompt_lens[i] - 1
98
+
99
if logits_processors:
100
found_logits_processors = True
101
for seq_id in seq_ids:
@@ -100,5 +108,6 @@ def _apply_logits_processors(
108
else:
109
logits_row_idx += len(seq_ids)
110
if found_logits_processors:
111
+ # verifies that no rows in logits were missed unexpectedly
112
assert logits_row_idx == logits.shape[0]
113
return logits