A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core] *Prompt* logprobs support in Multi-step (#8199)
afeldman-nm committed
Sep 18, 2024 at 11:38 UTC
a8c1d161a7d87dbc6c7cccfce303dcbe2e4ed6be
5 files changed
+300
-59
tests/conftest.py
+52
-32
@@ -20,6 +20,8 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BatchEncoding,
20
BatchFeature)
21
from transformers.models.auto.auto_factory import _BaseAutoModelClass
22
23
+from tests.models.utils import (TokensTextLogprobs,
24
+ TokensTextLogprobsPromptLogprobs)
25
from vllm import LLM, SamplingParams
26
from vllm.assets.image import ImageAsset
27
from vllm.assets.video import VideoAsset
@@ -33,7 +35,6 @@ from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
35
to_enc_dec_tuple_list, zip_enc_dec_prompts)
36
from vllm.logger import init_logger
37
from vllm.outputs import RequestOutput
36
-from vllm.sequence import SampleLogprobs
38
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
39
identity, is_cpu)
40
@@ -469,7 +470,7 @@ class HfRunner:
470
audios: Optional[PromptAudioInput] = None,
471
videos: Optional[List[np.ndarray]] = None,
472
**kwargs: Any,
472
- ) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
473
+ ) -> List[TokensTextLogprobs]:
474
all_logprobs: List[List[Dict[int, float]]] = []
475
all_output_ids: List[List[int]] = []
476
all_output_strs: List[str] = []
@@ -525,7 +526,7 @@ class HfRunner:
526
max_tokens: int,
527
num_logprobs: int,
528
**kwargs: Any,
528
- ) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
529
+ ) -> List[TokensTextLogprobs]:
530
'''
531
Greedy logprobs generation for vLLM encoder/decoder models
532
'''
@@ -653,14 +654,16 @@ class VllmRunner:
654
@staticmethod
655
def _final_steps_generate_w_logprobs(
656
req_outputs: List[RequestOutput],
656
- ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
657
- outputs: List[Tuple[List[int], str, Optional[SampleLogprobs]]] = []
657
+ ) -> List[TokensTextLogprobsPromptLogprobs]:
658
+ outputs: List[TokensTextLogprobsPromptLogprobs] = []
659
for req_output in req_outputs:
660
+ assert len(req_output.outputs) > 0
661
for sample in req_output.outputs:
662
output_str = sample.text
663
output_ids = list(sample.token_ids)
664
output_logprobs = sample.logprobs
663
- outputs.append((output_ids, output_str, output_logprobs))
665
+ outputs.append((output_ids, output_str, output_logprobs,
666
+ req_output.prompt_logprobs))
667
return outputs
668
669
def generate_w_logprobs(
@@ -670,7 +673,8 @@ class VllmRunner:
673
images: Optional[PromptImageInput] = None,
674
audios: Optional[PromptAudioInput] = None,
675
videos: Optional[PromptVideoInput] = None,
673
- ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
676
+ ) -> Union[List[TokensTextLogprobs],
677
+ List[TokensTextLogprobsPromptLogprobs]]:
678
assert sampling_params.logprobs is not None
679
680
if images is not None:
@@ -695,13 +699,20 @@ class VllmRunner:
699
700
req_outputs = self.model.generate(inputs,
701
sampling_params=sampling_params)
698
- return self._final_steps_generate_w_logprobs(req_outputs)
702
+
703
+ toks_str_logsprobs_prompt_logprobs = (
704
+ self._final_steps_generate_w_logprobs(req_outputs))
705
+ # Omit prompt logprobs if not required by sampling params
706
+ return ([x[0:-1] for x in toks_str_logsprobs_prompt_logprobs]
707
+ if sampling_params.prompt_logprobs is None else
708
+ toks_str_logsprobs_prompt_logprobs)
709
710
def generate_encoder_decoder_w_logprobs(
711
self,
712
encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
713
sampling_params: SamplingParams,
704
- ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
714
+ ) -> Union[List[TokensTextLogprobs],
715
+ List[TokensTextLogprobsPromptLogprobs]]:
716
'''
717
Logprobs generation for vLLM encoder/decoder models
718
'''
@@ -709,7 +720,12 @@ class VllmRunner:
720
assert sampling_params.logprobs is not None
721
req_outputs = self.model.generate(encoder_decoder_prompts,
722
sampling_params=sampling_params)
712
- return self._final_steps_generate_w_logprobs(req_outputs)
723
+ toks_str_logsprobs_prompt_logprobs = (
724
+ self._final_steps_generate_w_logprobs(req_outputs))
725
+ # Omit prompt logprobs if not required by sampling params
726
+ return ([x[0:-1] for x in toks_str_logsprobs_prompt_logprobs]
727
+ if sampling_params.prompt_logprobs is None else
728
+ toks_str_logsprobs_prompt_logprobs)
729
730
def generate_greedy(
731
self,
@@ -727,44 +743,48 @@ class VllmRunner:
743
prompts: List[str],
744
max_tokens: int,
745
num_logprobs: int,
746
+ num_prompt_logprobs: Optional[int] = None,
747
images: Optional[PromptImageInput] = None,
748
audios: Optional[PromptAudioInput] = None,
749
videos: Optional[PromptVideoInput] = None,
750
stop_token_ids: Optional[List[int]] = None,
734
- ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
735
- greedy_logprobs_params = SamplingParams(temperature=0.0,
736
- max_tokens=max_tokens,
737
- logprobs=num_logprobs,
738
- stop_token_ids=stop_token_ids)
739
- outputs = self.generate_w_logprobs(prompts,
740
- greedy_logprobs_params,
741
- images=images,
742
- audios=audios,
743
- videos=videos)
744
-
745
- return [(output_ids, output_str, output_logprobs)
746
- for output_ids, output_str, output_logprobs in outputs]
751
+ ) -> Union[List[TokensTextLogprobs],
752
+ List[TokensTextLogprobsPromptLogprobs]]:
753
+ greedy_logprobs_params = SamplingParams(
754
+ temperature=0.0,
755
+ max_tokens=max_tokens,
756
+ logprobs=num_logprobs,
757
+ prompt_logprobs=(num_prompt_logprobs),
758
+ stop_token_ids=stop_token_ids)
759
+
760
+ return self.generate_w_logprobs(prompts,
761
+ greedy_logprobs_params,
762
+ images=images,
763
+ audios=audios,
764
+ videos=videos)
765
766
def generate_encoder_decoder_greedy_logprobs(
767
self,
768
encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
769
max_tokens: int,
770
num_logprobs: int,
753
- ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
754
- greedy_logprobs_params = SamplingParams(temperature=0.0,
755
- use_beam_search=False,
756
- max_tokens=max_tokens,
757
- logprobs=num_logprobs)
771
+ num_prompt_logprobs: Optional[int] = None,
772
+ ) -> Union[List[TokensTextLogprobs],
773
+ List[TokensTextLogprobsPromptLogprobs]]:
774
+ greedy_logprobs_params = SamplingParams(
775
+ temperature=0.0,
776
+ use_beam_search=False,
777
+ max_tokens=max_tokens,
778
+ logprobs=num_logprobs,
779
+ prompt_logprobs=(num_prompt_logprobs),
780
+ )
781
'''
782
Greedy logprobs generation for vLLM encoder/decoder models
783
'''
784
762
- outputs = self.generate_encoder_decoder_w_logprobs(
785
+ return self.generate_encoder_decoder_w_logprobs(
786
encoder_decoder_prompts, greedy_logprobs_params)
787
765
- return [(output_ids, output_str, output_logprobs)
766
- for output_ids, output_str, output_logprobs in outputs]
767
-
788
def generate_beam_search(
789
self,
790
prompts: List[str],
tests/models/utils.py
+102
-6
@@ -1,7 +1,7 @@
1
import warnings
2
from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4
-from vllm.sequence import Logprob, SampleLogprobs
4
+from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
5
6
TokensText = Tuple[List[int], str]
7
@@ -34,20 +34,47 @@ def check_outputs_equal(
34
assert output_ids_0 == output_ids_1, fail_msg
35
36
37
+# Representation of generated sequence as a tuple of
38
+# * Token ID list
39
+# * String
40
+# * List of top sample logprobs for each sampled token
41
+#
42
+# Assumes prompt logprobs were not requested.
43
TokensTextLogprobs = Tuple[List[int], str, Optional[Union[List[Dict[int,
44
float]],
45
SampleLogprobs]]]
46
41
-# Allow for tokens to be represented as str's rather than IDs
47
+# Allow for tokens to be represented as str's rather than IDs;
48
+# tuple of
49
+# * Token string representations list
50
+# * String
51
+# * Optional list of top sample logprobs for each sampled token
52
+#
53
+# Assumes prompt logprobs were not requested.
54
TextTextLogprobs = Tuple[List[str], str, Optional[Union[List[Dict[str, float]],
55
List[Dict[str,
56
Logprob]]]]]
57
58
+# Representation of generated sequence as a tuple of
59
+# * Token ID list
60
+# * String
61
+# * Optional list of top sample logprobs for each sampled token
62
+# * Optional list of top prompt logprobs for each prompt token
63
+#
64
+# Allows prompt logprobs to be requested.
65
+TokensTextLogprobsPromptLogprobs = Tuple[
66
+ List[int], str, Optional[Union[List[Dict[int, float]], SampleLogprobs]],
67
+ Optional[Union[List[Optional[Dict[int, float]]], PromptLogprobs]]]
68
+
69
70
def check_logprobs_close(
71
*,
49
- outputs_0_lst: Sequence[Union[TokensTextLogprobs, TextTextLogprobs]],
50
- outputs_1_lst: Sequence[Union[TokensTextLogprobs, TextTextLogprobs]],
72
+ outputs_0_lst: Sequence[Union[TokensTextLogprobs,
73
+ TokensTextLogprobsPromptLogprobs,
74
+ TextTextLogprobs]],
75
+ outputs_1_lst: Sequence[Union[TokensTextLogprobs,
76
+ TokensTextLogprobsPromptLogprobs,
77
+ TextTextLogprobs]],
78
name_0: str,
79
name_1: str,
80
num_outputs_0_skip_tokens: int = 0,
@@ -57,6 +84,18 @@ def check_logprobs_close(
84
"""Compare the logprobs of two sequences generated by different models,
85
which should be similar but not necessarily equal.
86
87
+ How sample logprobs are compared:
88
+ * `always_check_logprobs == True`: set of highest-logprob token ids
89
+ must match between seq0 and seq1 at all sampled token offsets
90
+ * `always_check_logprobs == False`: highest-logprob token ids are
91
+ only compared at sampled token offsets for which generated token
92
+ ids don't match
93
+
94
+ Prompt logprobs must be provided either for both input sequences, or
95
+ for neither. If prompt logprobs are provided, then highest-logprob
96
+ prompt token ids must match between seq0 and seq1 at all prompt token
97
+ offsets.
98
+
99
Args:
100
outputs_0_lst: First sequence to compare
101
outputs_0_lst: Second sequence to compare
@@ -78,8 +117,65 @@ def check_logprobs_close(
117
for prompt_idx, (outputs_0,
118
outputs_1) in enumerate(zip(outputs_0_lst,
119
outputs_1_lst)):
81
- output_ids_0, output_str_0, logprobs_0 = outputs_0
82
- output_ids_1, output_str_1, logprobs_1 = outputs_1
120
+ assert len(outputs_0) == len(outputs_1)
121
+ if len(outputs_0) == 3:
122
+ assert len(outputs_1) == 3
123
+ # Break out tokens, text & sample logprobs
124
+ # (prompt logprobs were not provided)
125
+ output_ids_0, output_str_0, logprobs_0 = outputs_0
126
+ output_ids_1, output_str_1, logprobs_1 = outputs_1
127
+ elif len(outputs_0) == 4:
128
+ assert len(outputs_1) == 4
129
+ # Break out tokens, text, sample logprobs & prompt logprobs
130
+ (
131
+ output_ids_0,
132
+ output_str_0,
133
+ logprobs_0,
134
+ prompt_logprobs_0,
135
+ ) = outputs_0
136
+ (
137
+ output_ids_1,
138
+ output_str_1,
139
+ logprobs_1,
140
+ prompt_logprobs_1,
141
+ ) = outputs_1
142
+
143
+ # Test prompt logprobs closeness
144
+ if (prompt_logprobs_0 is not None
145
+ and prompt_logprobs_1 is not None):
146
+ # Both sequences' prompt logprobs lists are not `None``
147
+ # (although individual list elements may be `None`);
148
+ # for each token's logprobs:
149
+ for idx, (logprobs_elem_0, logprobs_elem_1) in enumerate(
150
+ zip(prompt_logprobs_0, prompt_logprobs_1)):
151
+ fail_msg = (
152
+ f"Prompt logprobs test:"
153
+ f"\n{name_0}:\tPrompt index {idx}\t{logprobs_elem_0}"
154
+ f"\n{name_1}:\tPrompt index {idx}\t{logprobs_elem_1}")
155
+
156
+ if logprobs_elem_0 is None:
157
+ # If the seq 0 token's logprobs are `None`,
158
+ # the seq 1 token's logprobs must be `None`
159
+ assert logprobs_elem_1 is None, fail_msg
160
+ else:
161
+ # If the seq 0 token's logprobs are not `None`,
162
+ # the seq 1 token's logprobs must not be `None`
163
+ assert logprobs_elem_1 is not None, fail_msg
164
+ # Logprobs check: top-k token choices must be the same
165
+ assert (set(logprobs_elem_0.keys()) == set(
166
+ logprobs_elem_1.keys())), fail_msg
167
+ else:
168
+ # Both sequence logprobs lists must be `None`
169
+ fail_msg = (f"Prompt logprobs test:"
170
+ f"\n{name_0}:\tlogprobs\t{prompt_logprobs_0}"
171
+ f"\n{name_1}:\tlogprobs\t{prompt_logprobs_1}")
172
+
173
+ assert (prompt_logprobs_0 is None
174
+ and prompt_logprobs_1 is None), fail_msg
175
+ else:
176
+ raise ValueError(f"Outputs tuple must have 3 or 4 elements but "
177
+ f"{len(outputs_0)} elements were provided: "
178
+ f"{outputs_0}")
179
180
if logprobs_0 is None:
181
logprobs_0 = [None] * len(output_ids_0)
tests/multi_step/test_correctness_llm.py
+92
@@ -100,3 +100,95 @@ def test_multi_step_llm(
100
name_0="hf",
101
name_1="vllm",
102
)
103
+
104
+
105
+@pytest.mark.parametrize("model", MODELS)
106
+@pytest.mark.parametrize("dtype", ["half"])
107
+@pytest.mark.parametrize("tp_size", [1])
108
+@pytest.mark.parametrize("max_tokens", [5])
109
+@pytest.mark.parametrize("enforce_eager", [True])
110
+@pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
111
+@pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
112
+@pytest.mark.parametrize("num_logprobs,num_prompt_logprobs", [(5, 5)])
113
+def test_multi_step_llm_w_prompt_logprobs(
114
+ vllm_runner,
115
+ example_prompts,
116
+ model: str,
117
+ dtype: str,
118
+ tp_size: int,
119
+ max_tokens: int,
120
+ enforce_eager: int,
121
+ num_scheduler_steps: int,
122
+ num_prompts: int,
123
+ num_logprobs: Optional[int],
124
+ num_prompt_logprobs: Optional[int],
125
+) -> None:
126
+ """Test prompt logprobs with multi-step scheduling via sync LLM Engine.
127
+
128
+ Set up a vLLM engine instance w/ single-step scheduling as a ground-truth
129
+ reference.
130
+
131
+ Prompt them with the same example prompts.
132
+
133
+ Validate:
134
+ * All generated logprobs are all very close
135
+
136
+ Args:
137
+ hf_runner: HF transformers model runner fixture
138
+ vllm_runner: vLLM model runner fixture
139
+ example_prompts: test fixture providing example prompts
140
+ model: model under test (same for single- and multi-step engines)
141
+ dtype: tensor datatype for engine to utilize
142
+ tp_size: degree of tensor-parallelism
143
+ max_tokens: the maximum number of tokens to generate
144
+ enforce_eager
145
+ num_scheduler_steps: for multi-step scheduling, GPU-side steps per
146
+ GPU -> CPU output transfer
147
+ num_prompts: number of example prompts under test
148
+ num_logprobs: corresponds to the `logprobs` argument to the OpenAI
149
+ completions endpoint; `None` -> no logprobs
150
+ num_prompt_logprobs: number of logprobs to return for each prompt token;
151
+ note that this argument is not supported by the
152
+ OpenAI completions endpoint.
153
+ """
154
+
155
+ prompts = example_prompts
156
+ if len(prompts) < num_prompts:
157
+ prompts = prompts * ((num_prompts // len(prompts)) + 1)
158
+ prompts = prompts[:num_prompts]
159
+ assert len(prompts) == num_prompts
160
+
161
+ with vllm_runner(
162
+ model,
163
+ dtype=dtype,
164
+ enforce_eager=enforce_eager,
165
+ gpu_memory_utilization=0.7,
166
+ tensor_parallel_size=tp_size,
167
+ use_v2_block_manager=True,
168
+ num_scheduler_steps=num_scheduler_steps,
169
+ ) as vllm_model:
170
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
171
+ prompts,
172
+ max_tokens,
173
+ num_logprobs,
174
+ num_prompt_logprobs=num_prompt_logprobs)
175
+
176
+ with vllm_runner(
177
+ model,
178
+ dtype=dtype,
179
+ enforce_eager=enforce_eager,
180
+ gpu_memory_utilization=0.7,
181
+ tensor_parallel_size=tp_size,
182
+ ) as vllm_model:
183
+ single_step_vllm_outputs = vllm_model.generate_greedy_logprobs(
184
+ prompts,
185
+ max_tokens,
186
+ num_logprobs,
187
+ num_prompt_logprobs=num_prompt_logprobs)
188
+
189
+ check_logprobs_close(
190
+ outputs_0_lst=single_step_vllm_outputs,
191
+ outputs_1_lst=vllm_outputs,
192
+ name_0="hf",
193
+ name_1="vllm",
194
+ )
tests/utils.py
+2
-1
@@ -493,6 +493,7 @@ async def completions_with_server_args(
493
'''
494
495
outputs = None
496
+ max_wait_seconds = 240 * 3 # 240 is default
497
with RemoteOpenAIServer(model_name,
498
server_cli_args,
499
max_wait_seconds=max_wait_seconds) as server:
@@ -503,7 +504,7 @@ async def completions_with_server_args(
504
stream=False,
505
max_tokens=5,
506
logprobs=num_logprobs)
506
- assert outputs is not None
507
+ assert outputs is not None, "Completion API call failed."
508
509
return outputs
510
vllm/worker/multi_step_model_runner.py
+52
-20
@@ -614,34 +614,66 @@ def _pythonize_sampler_output(
614
615
frozen_model_input = model_input.frozen_model_input
616
assert frozen_model_input.sampling_metadata is not None
617
+ sampling_metadata = frozen_model_input.sampling_metadata
618
# samples generation should have been skipped
619
assert not output.outputs
620
621
pinned_buffer = pinned_sampled_token_buffer[:model_input.num_queries]
622
622
- # CPU GPU sync
623
- pinned_buffer = pinned_buffer.copy_(sampled_token_ids, non_blocking=False)
623
+ # We guarantee output tensors are ready, so it is safe to
624
+ # pythonize the sampler output & obtain CPU-side logprobs.
625
+ #
626
+ # However we should check whether logprobs pythonization may
627
+ # be skipped entirely, i.e. because no logprobs were requested
628
+ # or pythonization was not deferred. To that end,
629
+ #
630
+ # * `prompt_logprobs_are_requested_for_prefill` signals that
631
+ # there are *any* prefill-phase requests which specify that
632
+ # prompt logprobs should be returned.
633
+ #
634
+ # * `any_logprobs_are_requested` signals that there are any
635
+ # requests which (1) specify that sample logprobs should be
636
+ # returned, or (2) are in the prefill phase AND specify that
637
+ # prompt logprobs should be returned.
638
+ #
639
+ # Later on, these flags cause adjustments to the pythonization
640
+ # process to accommodate logprobs.
641
+
642
+ seq_groups = sampling_metadata.seq_groups
643
+ prompt_logprobs_are_requested_for_prefill = any([
644
+ sg.sampling_params.prompt_logprobs is not None and sg.is_prompt
645
+ for sg in seq_groups
646
+ ])
647
+ any_logprobs_are_requested = (
648
+ prompt_logprobs_are_requested_for_prefill
649
+ or any([sg.sampling_params.logprobs is not None for sg in seq_groups]))
650
+
651
+ if prompt_logprobs_are_requested_for_prefill:
652
+ # CPU GPU sync, after gathering *only* sampled tokens (since
653
+ # requesting prompt logprobs leads `sampled_token_ids` to
654
+ # include prompt token ids in addition to sampled token ids.)
655
+ sample_idx_tensor = torch.tensor(
656
+ [sdx for sg in seq_groups for sdx in sg.sample_indices])
657
+ pinned_buffer = pinned_buffer.copy_(
658
+ sampled_token_ids[sample_idx_tensor, :], non_blocking=False)
659
+ else:
660
+ # CPU GPU sync
661
+ pinned_buffer = pinned_buffer.copy_(sampled_token_ids,
662
+ non_blocking=False)
663
664
# this will not block as the tensors are already on CPU
665
samples_list = pinned_buffer.tolist()
666
628
- sampling_metadata = frozen_model_input.sampling_metadata
629
-
667
skip_sampler_cpu_output = (
668
frozen_model_input.sampling_metadata.skip_sampler_cpu_output)
669
633
- # We are guaranteed output tensors are ready, so it is safe to
634
- # pythonize the sampler output & obtain CPU-side logprobs.
635
- #
636
- # However this computation may be skipped entirely
637
- # if no pythonization was deferred.
638
- seq_groups = sampling_metadata.seq_groups
639
- logprobs_are_requested = any([
640
- sg.sampling_params.logprobs is not None
641
- or sg.sampling_params.prompt_logprobs is not None for sg in seq_groups
642
- ])
670
+ # *Don't* skip logprobs pythonization *if*:
671
+ # * Any requests require logprobs to be returned in this
672
+ # iteration AND
673
+ # * These requests are being scheduled in a fashion which
674
+ # defers pythonization (i.e. multi-step scheduling.)
675
do_pythonize_logprobs = (skip_sampler_cpu_output
644
- and logprobs_are_requested)
676
+ and any_logprobs_are_requested)
677
(
678
prompt_logprobs,
679
sample_logprobs,
@@ -666,7 +698,7 @@ def _pythonize_sampler_output(
698
prompt_logprobs[sgdx],
699
sample_logprobs[sgdx],
700
)
669
- elif logprobs_are_requested:
701
+ elif any_logprobs_are_requested:
702
(
703
group_prompt_logprobs,
704
group_sample_logprobs,
@@ -696,7 +728,7 @@ def _pythonize_sampler_output(
728
seq_output.parent_seq_id = seq_ids[parent_id]
729
seq_output.output_token = next_token_id
730
699
- if logprobs_are_requested:
731
+ if any_logprobs_are_requested:
732
seq_output.logprobs = group_sample_logprobs[tdx]
733
else:
734
logprobs = next(iter(seq_output.logprobs.values()))
@@ -714,7 +746,7 @@ def _pythonize_sampler_output(
746
seq_outputs.append(
747
SequenceOutput(seq_ids[parent_id], next_token_id,
748
(group_sample_logprobs[tdx]
717
- if logprobs_are_requested else {
749
+ if any_logprobs_are_requested else {
750
next_token_id:
751
Logprob(logprob=float('inf'),
752
rank=None,
@@ -722,12 +754,12 @@ def _pythonize_sampler_output(
754
})))
755
if cache is not None:
756
completion_seq_group_output.prompt_logprobs = \
725
- group_prompt_logprobs if logprobs_are_requested else None
757
+ group_prompt_logprobs if any_logprobs_are_requested else None
758
output.outputs.append(completion_seq_group_output)
759
else:
760
output.outputs.append(
761
CompletionSequenceGroupOutput(
762
seq_outputs, (group_prompt_logprobs
731
- if logprobs_are_requested else None)))
763
+ if any_logprobs_are_requested else None)))
764
765
assert len(output.outputs) > 0