A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix] Fix prompt_logprobs when SamplingParams.detokenize is set to True (#5226)
zifeitong committed
Jun 4, 2024 at 19:37 UTC
974fc9b8455ec6c210534c176fd7d1245ca43261
2 files changed
+22
-13
tests/samplers/test_logprobs.py
+18
-9
@@ -12,6 +12,7 @@ MODELS = ["facebook/opt-125m"]
12
@pytest.mark.parametrize("dtype", ["half"])
13
@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
14
@pytest.mark.parametrize("num_top_logprobs", [6]) # 32000 == vocab_size
15
+@pytest.mark.parametrize("detokenize", [True, False])
16
def test_get_prompt_logprobs(
17
hf_runner,
18
vllm_runner,
@@ -19,6 +20,7 @@ def test_get_prompt_logprobs(
20
dtype,
21
chunked_prefill_token_size: int,
22
num_top_logprobs: int,
23
+ detokenize: bool,
24
example_prompts,
25
):
26
max_num_seqs = 256
@@ -48,7 +50,8 @@ def test_get_prompt_logprobs(
50
vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
51
logprobs=num_top_logprobs,
52
prompt_logprobs=num_top_logprobs,
51
- temperature=0.0)
53
+ temperature=0.0,
54
+ detokenize=detokenize)
55
vllm_results = vllm_model.model.generate(
56
example_prompts, sampling_params=vllm_sampling_params)
57
@@ -65,11 +68,16 @@ def test_get_prompt_logprobs(
68
top_logprob = next(iter(top_logprobs.values()))
69
output_string_from_most_likely_tokens.append(
70
top_logprob.decoded_token)
68
- output_string_from_most_likely_tokens = "".join(
69
- output_string_from_most_likely_tokens)
70
- assert output_text == output_string_from_most_likely_tokens, (
71
- "The output text from the top logprob for each token position "
72
- "should be the same as the output text in the result.")
71
+
72
+ if detokenize:
73
+ output_string_from_most_likely_tokens = "".join(
74
+ output_string_from_most_likely_tokens)
75
+ assert output_text == output_string_from_most_likely_tokens, (
76
+ "The output text from the top logprob for each token position "
77
+ "should be the same as the output text in the result.")
78
+ else:
79
+ assert output_text == ''
80
+ assert output_string_from_most_likely_tokens == [None] * max_tokens
81
82
# The first prompt logprob is always None
83
assert result.prompt_logprobs[0] is None
@@ -98,9 +106,10 @@ def test_get_prompt_logprobs(
106
hf_logprob[i][-1][token_id].item(),
107
atol=1e-2,
108
rtol=1e-2)
101
- assert isinstance(sample_logprob.decoded_token, str), (
102
- "The token should be decoded by the time it is returned "
103
- " to the user.")
109
+ if detokenize:
110
+ assert isinstance(sample_logprob.decoded_token, str), (
111
+ "The token should be decoded by the time it is returned"
112
+ " to the user.")
113
114
# Test if prompt logprobs are correctly set.
115
for vllm_result in vllm_results:
vllm/engine/output_processor/single_step.py
+4
-4
@@ -60,10 +60,10 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
60
assert len(outputs) == 1, ("Single step should only has 1 output.")
61
output = outputs[0]
62
prompt_logprobs = output.prompt_logprobs
63
- if (prompt_logprobs is not None
64
- and seq_group.sampling_params.detokenize and self.detokenizer):
65
- self.detokenizer.decode_prompt_logprobs_inplace(
66
- seq_group, prompt_logprobs)
63
+ if prompt_logprobs is not None:
64
+ if seq_group.sampling_params.detokenize and self.detokenizer:
65
+ self.detokenizer.decode_prompt_logprobs_inplace(
66
+ seq_group, prompt_logprobs)
67
if not seq_group.prompt_logprobs:
68
# The first prompt token's logprob is None because it doesn't
69
# have tokens that are precedent.