@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 974fc9b845
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix] Fix prompt_logprobs when SamplingParams.detokenize is set to True (#5226)

zifeitong committed Jun 4, 2024 at 19:37 UTC 974fc9b8455ec6c210534c176fd7d1245ca43261
2 files changed +22 -13
tests/samplers/test_logprobs.py
+18 -9
@@ -12,6 +12,7 @@ MODELS = ["facebook/opt-125m"]
12 @pytest.mark.parametrize("dtype", ["half"])
13 @pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
14 @pytest.mark.parametrize("num_top_logprobs", [6]) # 32000 == vocab_size
15 +@pytest.mark.parametrize("detokenize", [True, False])
16 def test_get_prompt_logprobs(
17 hf_runner,
18 vllm_runner,
@@ -19,6 +20,7 @@ def test_get_prompt_logprobs(
20 dtype,
21 chunked_prefill_token_size: int,
22 num_top_logprobs: int,
23 + detokenize: bool,
24 example_prompts,
25 ):
26 max_num_seqs = 256
@@ -48,7 +50,8 @@ def test_get_prompt_logprobs(
50 vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
51 logprobs=num_top_logprobs,
52 prompt_logprobs=num_top_logprobs,
51 - temperature=0.0)
53 + temperature=0.0,
54 + detokenize=detokenize)
55 vllm_results = vllm_model.model.generate(
56 example_prompts, sampling_params=vllm_sampling_params)
57
@@ -65,11 +68,16 @@ def test_get_prompt_logprobs(
68 top_logprob = next(iter(top_logprobs.values()))
69 output_string_from_most_likely_tokens.append(
70 top_logprob.decoded_token)
68 - output_string_from_most_likely_tokens = "".join(
69 - output_string_from_most_likely_tokens)
70 - assert output_text == output_string_from_most_likely_tokens, (
71 - "The output text from the top logprob for each token position "
72 - "should be the same as the output text in the result.")
71 +
72 + if detokenize:
73 + output_string_from_most_likely_tokens = "".join(
74 + output_string_from_most_likely_tokens)
75 + assert output_text == output_string_from_most_likely_tokens, (
76 + "The output text from the top logprob for each token position "
77 + "should be the same as the output text in the result.")
78 + else:
79 + assert output_text == ''
80 + assert output_string_from_most_likely_tokens == [None] * max_tokens
81
82 # The first prompt logprob is always None
83 assert result.prompt_logprobs[0] is None
@@ -98,9 +106,10 @@ def test_get_prompt_logprobs(
106 hf_logprob[i][-1][token_id].item(),
107 atol=1e-2,
108 rtol=1e-2)
101 - assert isinstance(sample_logprob.decoded_token, str), (
102 - "The token should be decoded by the time it is returned "
103 - " to the user.")
109 + if detokenize:
110 + assert isinstance(sample_logprob.decoded_token, str), (
111 + "The token should be decoded by the time it is returned"
112 + " to the user.")
113
114 # Test if prompt logprobs are correctly set.
115 for vllm_result in vllm_results:
vllm/engine/output_processor/single_step.py
+4 -4
@@ -60,10 +60,10 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
60 assert len(outputs) == 1, ("Single step should only has 1 output.")
61 output = outputs[0]
62 prompt_logprobs = output.prompt_logprobs
63 - if (prompt_logprobs is not None
64 - and seq_group.sampling_params.detokenize and self.detokenizer):
65 - self.detokenizer.decode_prompt_logprobs_inplace(
66 - seq_group, prompt_logprobs)
63 + if prompt_logprobs is not None:
64 + if seq_group.sampling_params.detokenize and self.detokenizer:
65 + self.detokenizer.decode_prompt_logprobs_inplace(
66 + seq_group, prompt_logprobs)
67 if not seq_group.prompt_logprobs:
68 # The first prompt token's logprob is None because it doesn't
69 # have tokens that are precedent.