@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a8c1d161a7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core] *Prompt* logprobs support in Multi-step (#8199)

afeldman-nm committed Sep 18, 2024 at 11:38 UTC a8c1d161a7d87dbc6c7cccfce303dcbe2e4ed6be
5 files changed +300 -59
tests/conftest.py
+52 -32
@@ -20,6 +20,8 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BatchEncoding,
20 BatchFeature)
21 from transformers.models.auto.auto_factory import _BaseAutoModelClass
22
23 +from tests.models.utils import (TokensTextLogprobs,
24 + TokensTextLogprobsPromptLogprobs)
25 from vllm import LLM, SamplingParams
26 from vllm.assets.image import ImageAsset
27 from vllm.assets.video import VideoAsset
@@ -33,7 +35,6 @@ from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
35 to_enc_dec_tuple_list, zip_enc_dec_prompts)
36 from vllm.logger import init_logger
37 from vllm.outputs import RequestOutput
36 -from vllm.sequence import SampleLogprobs
38 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
39 identity, is_cpu)
40
@@ -469,7 +470,7 @@ class HfRunner:
470 audios: Optional[PromptAudioInput] = None,
471 videos: Optional[List[np.ndarray]] = None,
472 **kwargs: Any,
472 - ) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
473 + ) -> List[TokensTextLogprobs]:
474 all_logprobs: List[List[Dict[int, float]]] = []
475 all_output_ids: List[List[int]] = []
476 all_output_strs: List[str] = []
@@ -525,7 +526,7 @@ class HfRunner:
526 max_tokens: int,
527 num_logprobs: int,
528 **kwargs: Any,
528 - ) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
529 + ) -> List[TokensTextLogprobs]:
530 '''
531 Greedy logprobs generation for vLLM encoder/decoder models
532 '''
@@ -653,14 +654,16 @@ class VllmRunner:
654 @staticmethod
655 def _final_steps_generate_w_logprobs(
656 req_outputs: List[RequestOutput],
656 - ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
657 - outputs: List[Tuple[List[int], str, Optional[SampleLogprobs]]] = []
657 + ) -> List[TokensTextLogprobsPromptLogprobs]:
658 + outputs: List[TokensTextLogprobsPromptLogprobs] = []
659 for req_output in req_outputs:
660 + assert len(req_output.outputs) > 0
661 for sample in req_output.outputs:
662 output_str = sample.text
663 output_ids = list(sample.token_ids)
664 output_logprobs = sample.logprobs
663 - outputs.append((output_ids, output_str, output_logprobs))
665 + outputs.append((output_ids, output_str, output_logprobs,
666 + req_output.prompt_logprobs))
667 return outputs
668
669 def generate_w_logprobs(
@@ -670,7 +673,8 @@ class VllmRunner:
673 images: Optional[PromptImageInput] = None,
674 audios: Optional[PromptAudioInput] = None,
675 videos: Optional[PromptVideoInput] = None,
673 - ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
676 + ) -> Union[List[TokensTextLogprobs],
677 + List[TokensTextLogprobsPromptLogprobs]]:
678 assert sampling_params.logprobs is not None
679
680 if images is not None:
@@ -695,13 +699,20 @@ class VllmRunner:
699
700 req_outputs = self.model.generate(inputs,
701 sampling_params=sampling_params)
698 - return self._final_steps_generate_w_logprobs(req_outputs)
702 +
703 + toks_str_logsprobs_prompt_logprobs = (
704 + self._final_steps_generate_w_logprobs(req_outputs))
705 + # Omit prompt logprobs if not required by sampling params
706 + return ([x[0:-1] for x in toks_str_logsprobs_prompt_logprobs]
707 + if sampling_params.prompt_logprobs is None else
708 + toks_str_logsprobs_prompt_logprobs)
709
710 def generate_encoder_decoder_w_logprobs(
711 self,
712 encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
713 sampling_params: SamplingParams,
704 - ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
714 + ) -> Union[List[TokensTextLogprobs],
715 + List[TokensTextLogprobsPromptLogprobs]]:
716 '''
717 Logprobs generation for vLLM encoder/decoder models
718 '''
@@ -709,7 +720,12 @@ class VllmRunner:
720 assert sampling_params.logprobs is not None
721 req_outputs = self.model.generate(encoder_decoder_prompts,
722 sampling_params=sampling_params)
712 - return self._final_steps_generate_w_logprobs(req_outputs)
723 + toks_str_logsprobs_prompt_logprobs = (
724 + self._final_steps_generate_w_logprobs(req_outputs))
725 + # Omit prompt logprobs if not required by sampling params
726 + return ([x[0:-1] for x in toks_str_logsprobs_prompt_logprobs]
727 + if sampling_params.prompt_logprobs is None else
728 + toks_str_logsprobs_prompt_logprobs)
729
730 def generate_greedy(
731 self,
@@ -727,44 +743,48 @@ class VllmRunner:
743 prompts: List[str],
744 max_tokens: int,
745 num_logprobs: int,
746 + num_prompt_logprobs: Optional[int] = None,
747 images: Optional[PromptImageInput] = None,
748 audios: Optional[PromptAudioInput] = None,
749 videos: Optional[PromptVideoInput] = None,
750 stop_token_ids: Optional[List[int]] = None,
734 - ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
735 - greedy_logprobs_params = SamplingParams(temperature=0.0,
736 - max_tokens=max_tokens,
737 - logprobs=num_logprobs,
738 - stop_token_ids=stop_token_ids)
739 - outputs = self.generate_w_logprobs(prompts,
740 - greedy_logprobs_params,
741 - images=images,
742 - audios=audios,
743 - videos=videos)
744 -
745 - return [(output_ids, output_str, output_logprobs)
746 - for output_ids, output_str, output_logprobs in outputs]
751 + ) -> Union[List[TokensTextLogprobs],
752 + List[TokensTextLogprobsPromptLogprobs]]:
753 + greedy_logprobs_params = SamplingParams(
754 + temperature=0.0,
755 + max_tokens=max_tokens,
756 + logprobs=num_logprobs,
757 + prompt_logprobs=(num_prompt_logprobs),
758 + stop_token_ids=stop_token_ids)
759 +
760 + return self.generate_w_logprobs(prompts,
761 + greedy_logprobs_params,
762 + images=images,
763 + audios=audios,
764 + videos=videos)
765
766 def generate_encoder_decoder_greedy_logprobs(
767 self,
768 encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
769 max_tokens: int,
770 num_logprobs: int,
753 - ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
754 - greedy_logprobs_params = SamplingParams(temperature=0.0,
755 - use_beam_search=False,
756 - max_tokens=max_tokens,
757 - logprobs=num_logprobs)
771 + num_prompt_logprobs: Optional[int] = None,
772 + ) -> Union[List[TokensTextLogprobs],
773 + List[TokensTextLogprobsPromptLogprobs]]:
774 + greedy_logprobs_params = SamplingParams(
775 + temperature=0.0,
776 + use_beam_search=False,
777 + max_tokens=max_tokens,
778 + logprobs=num_logprobs,
779 + prompt_logprobs=(num_prompt_logprobs),
780 + )
781 '''
782 Greedy logprobs generation for vLLM encoder/decoder models
783 '''
784
762 - outputs = self.generate_encoder_decoder_w_logprobs(
785 + return self.generate_encoder_decoder_w_logprobs(
786 encoder_decoder_prompts, greedy_logprobs_params)
787
765 - return [(output_ids, output_str, output_logprobs)
766 - for output_ids, output_str, output_logprobs in outputs]
767 -
788 def generate_beam_search(
789 self,
790 prompts: List[str],
tests/models/utils.py
+102 -6
@@ -1,7 +1,7 @@
1 import warnings
2 from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4 -from vllm.sequence import Logprob, SampleLogprobs
4 +from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
5
6 TokensText = Tuple[List[int], str]
7
@@ -34,20 +34,47 @@ def check_outputs_equal(
34 assert output_ids_0 == output_ids_1, fail_msg
35
36
37 +# Representation of generated sequence as a tuple of
38 +# * Token ID list
39 +# * String
40 +# * List of top sample logprobs for each sampled token
41 +#
42 +# Assumes prompt logprobs were not requested.
43 TokensTextLogprobs = Tuple[List[int], str, Optional[Union[List[Dict[int,
44 float]],
45 SampleLogprobs]]]
46
41 -# Allow for tokens to be represented as str's rather than IDs
47 +# Allow for tokens to be represented as str's rather than IDs;
48 +# tuple of
49 +# * Token string representations list
50 +# * String
51 +# * Optional list of top sample logprobs for each sampled token
52 +#
53 +# Assumes prompt logprobs were not requested.
54 TextTextLogprobs = Tuple[List[str], str, Optional[Union[List[Dict[str, float]],
55 List[Dict[str,
56 Logprob]]]]]
57
58 +# Representation of generated sequence as a tuple of
59 +# * Token ID list
60 +# * String
61 +# * Optional list of top sample logprobs for each sampled token
62 +# * Optional list of top prompt logprobs for each prompt token
63 +#
64 +# Allows prompt logprobs to be requested.
65 +TokensTextLogprobsPromptLogprobs = Tuple[
66 + List[int], str, Optional[Union[List[Dict[int, float]], SampleLogprobs]],
67 + Optional[Union[List[Optional[Dict[int, float]]], PromptLogprobs]]]
68 +
69
70 def check_logprobs_close(
71 *,
49 - outputs_0_lst: Sequence[Union[TokensTextLogprobs, TextTextLogprobs]],
50 - outputs_1_lst: Sequence[Union[TokensTextLogprobs, TextTextLogprobs]],
72 + outputs_0_lst: Sequence[Union[TokensTextLogprobs,
73 + TokensTextLogprobsPromptLogprobs,
74 + TextTextLogprobs]],
75 + outputs_1_lst: Sequence[Union[TokensTextLogprobs,
76 + TokensTextLogprobsPromptLogprobs,
77 + TextTextLogprobs]],
78 name_0: str,
79 name_1: str,
80 num_outputs_0_skip_tokens: int = 0,
@@ -57,6 +84,18 @@ def check_logprobs_close(
84 """Compare the logprobs of two sequences generated by different models,
85 which should be similar but not necessarily equal.
86
87 + How sample logprobs are compared:
88 + * `always_check_logprobs == True`: set of highest-logprob token ids
89 + must match between seq0 and seq1 at all sampled token offsets
90 + * `always_check_logprobs == False`: highest-logprob token ids are
91 + only compared at sampled token offsets for which generated token
92 + ids don't match
93 +
94 + Prompt logprobs must be provided either for both input sequences, or
95 + for neither. If prompt logprobs are provided, then highest-logprob
96 + prompt token ids must match between seq0 and seq1 at all prompt token
97 + offsets.
98 +
99 Args:
100 outputs_0_lst: First sequence to compare
101 outputs_0_lst: Second sequence to compare
@@ -78,8 +117,65 @@ def check_logprobs_close(
117 for prompt_idx, (outputs_0,
118 outputs_1) in enumerate(zip(outputs_0_lst,
119 outputs_1_lst)):
81 - output_ids_0, output_str_0, logprobs_0 = outputs_0
82 - output_ids_1, output_str_1, logprobs_1 = outputs_1
120 + assert len(outputs_0) == len(outputs_1)
121 + if len(outputs_0) == 3:
122 + assert len(outputs_1) == 3
123 + # Break out tokens, text & sample logprobs
124 + # (prompt logprobs were not provided)
125 + output_ids_0, output_str_0, logprobs_0 = outputs_0
126 + output_ids_1, output_str_1, logprobs_1 = outputs_1
127 + elif len(outputs_0) == 4:
128 + assert len(outputs_1) == 4
129 + # Break out tokens, text, sample logprobs & prompt logprobs
130 + (
131 + output_ids_0,
132 + output_str_0,
133 + logprobs_0,
134 + prompt_logprobs_0,
135 + ) = outputs_0
136 + (
137 + output_ids_1,
138 + output_str_1,
139 + logprobs_1,
140 + prompt_logprobs_1,
141 + ) = outputs_1
142 +
143 + # Test prompt logprobs closeness
144 + if (prompt_logprobs_0 is not None
145 + and prompt_logprobs_1 is not None):
146 + # Both sequences' prompt logprobs lists are not `None``
147 + # (although individual list elements may be `None`);
148 + # for each token's logprobs:
149 + for idx, (logprobs_elem_0, logprobs_elem_1) in enumerate(
150 + zip(prompt_logprobs_0, prompt_logprobs_1)):
151 + fail_msg = (
152 + f"Prompt logprobs test:"
153 + f"\n{name_0}:\tPrompt index {idx}\t{logprobs_elem_0}"
154 + f"\n{name_1}:\tPrompt index {idx}\t{logprobs_elem_1}")
155 +
156 + if logprobs_elem_0 is None:
157 + # If the seq 0 token's logprobs are `None`,
158 + # the seq 1 token's logprobs must be `None`
159 + assert logprobs_elem_1 is None, fail_msg
160 + else:
161 + # If the seq 0 token's logprobs are not `None`,
162 + # the seq 1 token's logprobs must not be `None`
163 + assert logprobs_elem_1 is not None, fail_msg
164 + # Logprobs check: top-k token choices must be the same
165 + assert (set(logprobs_elem_0.keys()) == set(
166 + logprobs_elem_1.keys())), fail_msg
167 + else:
168 + # Both sequence logprobs lists must be `None`
169 + fail_msg = (f"Prompt logprobs test:"
170 + f"\n{name_0}:\tlogprobs\t{prompt_logprobs_0}"
171 + f"\n{name_1}:\tlogprobs\t{prompt_logprobs_1}")
172 +
173 + assert (prompt_logprobs_0 is None
174 + and prompt_logprobs_1 is None), fail_msg
175 + else:
176 + raise ValueError(f"Outputs tuple must have 3 or 4 elements but "
177 + f"{len(outputs_0)} elements were provided: "
178 + f"{outputs_0}")
179
180 if logprobs_0 is None:
181 logprobs_0 = [None] * len(output_ids_0)
tests/multi_step/test_correctness_llm.py
+92
@@ -100,3 +100,95 @@ def test_multi_step_llm(
100 name_0="hf",
101 name_1="vllm",
102 )
103 +
104 +
105 +@pytest.mark.parametrize("model", MODELS)
106 +@pytest.mark.parametrize("dtype", ["half"])
107 +@pytest.mark.parametrize("tp_size", [1])
108 +@pytest.mark.parametrize("max_tokens", [5])
109 +@pytest.mark.parametrize("enforce_eager", [True])
110 +@pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
111 +@pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
112 +@pytest.mark.parametrize("num_logprobs,num_prompt_logprobs", [(5, 5)])
113 +def test_multi_step_llm_w_prompt_logprobs(
114 + vllm_runner,
115 + example_prompts,
116 + model: str,
117 + dtype: str,
118 + tp_size: int,
119 + max_tokens: int,
120 + enforce_eager: int,
121 + num_scheduler_steps: int,
122 + num_prompts: int,
123 + num_logprobs: Optional[int],
124 + num_prompt_logprobs: Optional[int],
125 +) -> None:
126 + """Test prompt logprobs with multi-step scheduling via sync LLM Engine.
127 +
128 + Set up a vLLM engine instance w/ single-step scheduling as a ground-truth
129 + reference.
130 +
131 + Prompt them with the same example prompts.
132 +
133 + Validate:
134 + * All generated logprobs are all very close
135 +
136 + Args:
137 + hf_runner: HF transformers model runner fixture
138 + vllm_runner: vLLM model runner fixture
139 + example_prompts: test fixture providing example prompts
140 + model: model under test (same for single- and multi-step engines)
141 + dtype: tensor datatype for engine to utilize
142 + tp_size: degree of tensor-parallelism
143 + max_tokens: the maximum number of tokens to generate
144 + enforce_eager
145 + num_scheduler_steps: for multi-step scheduling, GPU-side steps per
146 + GPU -> CPU output transfer
147 + num_prompts: number of example prompts under test
148 + num_logprobs: corresponds to the `logprobs` argument to the OpenAI
149 + completions endpoint; `None` -> no logprobs
150 + num_prompt_logprobs: number of logprobs to return for each prompt token;
151 + note that this argument is not supported by the
152 + OpenAI completions endpoint.
153 + """
154 +
155 + prompts = example_prompts
156 + if len(prompts) < num_prompts:
157 + prompts = prompts * ((num_prompts // len(prompts)) + 1)
158 + prompts = prompts[:num_prompts]
159 + assert len(prompts) == num_prompts
160 +
161 + with vllm_runner(
162 + model,
163 + dtype=dtype,
164 + enforce_eager=enforce_eager,
165 + gpu_memory_utilization=0.7,
166 + tensor_parallel_size=tp_size,
167 + use_v2_block_manager=True,
168 + num_scheduler_steps=num_scheduler_steps,
169 + ) as vllm_model:
170 + vllm_outputs = vllm_model.generate_greedy_logprobs(
171 + prompts,
172 + max_tokens,
173 + num_logprobs,
174 + num_prompt_logprobs=num_prompt_logprobs)
175 +
176 + with vllm_runner(
177 + model,
178 + dtype=dtype,
179 + enforce_eager=enforce_eager,
180 + gpu_memory_utilization=0.7,
181 + tensor_parallel_size=tp_size,
182 + ) as vllm_model:
183 + single_step_vllm_outputs = vllm_model.generate_greedy_logprobs(
184 + prompts,
185 + max_tokens,
186 + num_logprobs,
187 + num_prompt_logprobs=num_prompt_logprobs)
188 +
189 + check_logprobs_close(
190 + outputs_0_lst=single_step_vllm_outputs,
191 + outputs_1_lst=vllm_outputs,
192 + name_0="hf",
193 + name_1="vllm",
194 + )
tests/utils.py
+2 -1
@@ -493,6 +493,7 @@ async def completions_with_server_args(
493 '''
494
495 outputs = None
496 + max_wait_seconds = 240 * 3 # 240 is default
497 with RemoteOpenAIServer(model_name,
498 server_cli_args,
499 max_wait_seconds=max_wait_seconds) as server:
@@ -503,7 +504,7 @@ async def completions_with_server_args(
504 stream=False,
505 max_tokens=5,
506 logprobs=num_logprobs)
506 - assert outputs is not None
507 + assert outputs is not None, "Completion API call failed."
508
509 return outputs
510
vllm/worker/multi_step_model_runner.py
+52 -20
@@ -614,34 +614,66 @@ def _pythonize_sampler_output(
614
615 frozen_model_input = model_input.frozen_model_input
616 assert frozen_model_input.sampling_metadata is not None
617 + sampling_metadata = frozen_model_input.sampling_metadata
618 # samples generation should have been skipped
619 assert not output.outputs
620
621 pinned_buffer = pinned_sampled_token_buffer[:model_input.num_queries]
622
622 - # CPU GPU sync
623 - pinned_buffer = pinned_buffer.copy_(sampled_token_ids, non_blocking=False)
623 + # We guarantee output tensors are ready, so it is safe to
624 + # pythonize the sampler output & obtain CPU-side logprobs.
625 + #
626 + # However we should check whether logprobs pythonization may
627 + # be skipped entirely, i.e. because no logprobs were requested
628 + # or pythonization was not deferred. To that end,
629 + #
630 + # * `prompt_logprobs_are_requested_for_prefill` signals that
631 + # there are *any* prefill-phase requests which specify that
632 + # prompt logprobs should be returned.
633 + #
634 + # * `any_logprobs_are_requested` signals that there are any
635 + # requests which (1) specify that sample logprobs should be
636 + # returned, or (2) are in the prefill phase AND specify that
637 + # prompt logprobs should be returned.
638 + #
639 + # Later on, these flags cause adjustments to the pythonization
640 + # process to accommodate logprobs.
641 +
642 + seq_groups = sampling_metadata.seq_groups
643 + prompt_logprobs_are_requested_for_prefill = any([
644 + sg.sampling_params.prompt_logprobs is not None and sg.is_prompt
645 + for sg in seq_groups
646 + ])
647 + any_logprobs_are_requested = (
648 + prompt_logprobs_are_requested_for_prefill
649 + or any([sg.sampling_params.logprobs is not None for sg in seq_groups]))
650 +
651 + if prompt_logprobs_are_requested_for_prefill:
652 + # CPU GPU sync, after gathering *only* sampled tokens (since
653 + # requesting prompt logprobs leads `sampled_token_ids` to
654 + # include prompt token ids in addition to sampled token ids.)
655 + sample_idx_tensor = torch.tensor(
656 + [sdx for sg in seq_groups for sdx in sg.sample_indices])
657 + pinned_buffer = pinned_buffer.copy_(
658 + sampled_token_ids[sample_idx_tensor, :], non_blocking=False)
659 + else:
660 + # CPU GPU sync
661 + pinned_buffer = pinned_buffer.copy_(sampled_token_ids,
662 + non_blocking=False)
663
664 # this will not block as the tensors are already on CPU
665 samples_list = pinned_buffer.tolist()
666
628 - sampling_metadata = frozen_model_input.sampling_metadata
629 -
667 skip_sampler_cpu_output = (
668 frozen_model_input.sampling_metadata.skip_sampler_cpu_output)
669
633 - # We are guaranteed output tensors are ready, so it is safe to
634 - # pythonize the sampler output & obtain CPU-side logprobs.
635 - #
636 - # However this computation may be skipped entirely
637 - # if no pythonization was deferred.
638 - seq_groups = sampling_metadata.seq_groups
639 - logprobs_are_requested = any([
640 - sg.sampling_params.logprobs is not None
641 - or sg.sampling_params.prompt_logprobs is not None for sg in seq_groups
642 - ])
670 + # *Don't* skip logprobs pythonization *if*:
671 + # * Any requests require logprobs to be returned in this
672 + # iteration AND
673 + # * These requests are being scheduled in a fashion which
674 + # defers pythonization (i.e. multi-step scheduling.)
675 do_pythonize_logprobs = (skip_sampler_cpu_output
644 - and logprobs_are_requested)
676 + and any_logprobs_are_requested)
677 (
678 prompt_logprobs,
679 sample_logprobs,
@@ -666,7 +698,7 @@ def _pythonize_sampler_output(
698 prompt_logprobs[sgdx],
699 sample_logprobs[sgdx],
700 )
669 - elif logprobs_are_requested:
701 + elif any_logprobs_are_requested:
702 (
703 group_prompt_logprobs,
704 group_sample_logprobs,
@@ -696,7 +728,7 @@ def _pythonize_sampler_output(
728 seq_output.parent_seq_id = seq_ids[parent_id]
729 seq_output.output_token = next_token_id
730
699 - if logprobs_are_requested:
731 + if any_logprobs_are_requested:
732 seq_output.logprobs = group_sample_logprobs[tdx]
733 else:
734 logprobs = next(iter(seq_output.logprobs.values()))
@@ -714,7 +746,7 @@ def _pythonize_sampler_output(
746 seq_outputs.append(
747 SequenceOutput(seq_ids[parent_id], next_token_id,
748 (group_sample_logprobs[tdx]
717 - if logprobs_are_requested else {
749 + if any_logprobs_are_requested else {
750 next_token_id:
751 Logprob(logprob=float('inf'),
752 rank=None,
@@ -722,12 +754,12 @@ def _pythonize_sampler_output(
754 })))
755 if cache is not None:
756 completion_seq_group_output.prompt_logprobs = \
725 - group_prompt_logprobs if logprobs_are_requested else None
757 + group_prompt_logprobs if any_logprobs_are_requested else None
758 output.outputs.append(completion_seq_group_output)
759 else:
760 output.outputs.append(
761 CompletionSequenceGroupOutput(
762 seq_outputs, (group_prompt_logprobs
731 - if logprobs_are_requested else None)))
763 + if any_logprobs_are_requested else None)))
764
765 assert len(output.outputs) > 0