@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f942efb5a3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Dynamic Spec Decoding] Auto-disable by the running queue size (#4592)

Co-authored-by: Cade Daniel <edacih@gmail.com>

Cody Yu committed May 8, 2024 at 14:44 UTC f942efb5a3712498b8b583d2d9345f98d15f22f0
11 files changed +227 -39
tests/samplers/test_rejection_sampler.py
+9 -4
@@ -42,9 +42,11 @@ def mock_causal_accepted_tensor(
42 @pytest.mark.parametrize(
43 "which_tokens_accepted",
44 ["all_tokens_accepted", "no_tokens_accepted", "some_tokens_accepted"])
45 +@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
46 @pytest.mark.parametrize("device", CUDA_DEVICES)
47 @torch.inference_mode()
47 -def test_correct_output_format(which_tokens_accepted: str, seed: int,
48 +def test_correct_output_format(which_tokens_accepted: str,
49 + disable_bonus_tokens: bool, seed: int,
50 device: str):
51 """Verify the output has correct format given predetermined accepted matrix.
52 """
@@ -82,7 +84,8 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
84 size=(batch_size, 1),
85 dtype=torch.int64)
86
85 - rejection_sampler = RejectionSampler()
87 + rejection_sampler = RejectionSampler(
88 + disable_bonus_tokens=disable_bonus_tokens)
89 rejection_sampler.init_gpu_tensors(rank=0)
90 output_token_ids = rejection_sampler._create_output( # pylint: disable=protected-access
91 accepted,
@@ -91,9 +94,11 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
94 bonus_token_ids,
95 )
96
94 - # Bonus tokens are currently disabled. Verify they're set to -1.
97 + expected_bonus_token_ids = bonus_token_ids.clone()
98 + # If bonus tokens disabled. Verify they are set to -1.
99 # See https://github.com/vllm-project/vllm/issues/4212
96 - expected_bonus_token_ids = bonus_token_ids.clone() * 0 - 1
100 + if disable_bonus_tokens:
101 + expected_bonus_token_ids = expected_bonus_token_ids * 0 - 1
102
103 if which_tokens_accepted == "all_tokens_accepted":
104 # Expect all tokens to be equal to draft tokens.
tests/spec_decode/e2e/test_multistep_correctness.py
+34
@@ -536,6 +536,40 @@ def test_skip_speculation(baseline_llm_generator, test_llm_generator,
536 force_output_len=True)
537
538
539 +@pytest.mark.parametrize(
540 + "common_llm_kwargs",
541 + [{
542 + "model": "JackFram/llama-160m",
543 +
544 + # Skip cuda graph recording for fast test.
545 + "enforce_eager": True,
546 +
547 + # Required for spec decode.
548 + "use_v2_block_manager": True
549 + }])
550 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
551 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
552 +@pytest.mark.parametrize("test_llm_kwargs", [
553 + {
554 + "speculative_model": "JackFram/llama-68m",
555 + "num_speculative_tokens": 5,
556 + "speculative_disable_by_batch_size": 2,
557 + },
558 +])
559 +@pytest.mark.parametrize("batch_size", [8])
560 +@pytest.mark.parametrize("output_len", [10])
561 +@pytest.mark.parametrize("seed", [1])
562 +def test_disable_speculation(baseline_llm_generator, test_llm_generator,
563 + batch_size: int, output_len: int):
564 + """Verify greedy equality when all sequences disable speculation.
565 + """
566 + run_greedy_equality_correctness_test(baseline_llm_generator,
567 + test_llm_generator,
568 + batch_size,
569 + max_output_len=output_len,
570 + force_output_len=True)
571 +
572 +
573 @pytest.mark.parametrize(
574 "common_llm_kwargs",
575 [{
tests/spec_decode/e2e/test_ngram_correctness.py
+1 -1
@@ -57,7 +57,7 @@ from .conftest import run_greedy_equality_correctness_test
57 @pytest.mark.parametrize("output_len", [
58 256,
59 ])
60 -@pytest.mark.parametrize("batch_size", [1, 64])
60 +@pytest.mark.parametrize("batch_size", [1, 32])
61 @pytest.mark.parametrize("seed", [1])
62 def test_ngram_e2e_greedy_correctness(baseline_llm_generator,
63 test_llm_generator, batch_size: int,
tests/spec_decode/test_dynamic_spec_decode.py new
+77
@@ -0,0 +1,77 @@
1 +from unittest.mock import MagicMock
2 +
3 +import pytest
4 +import torch
5 +
6 +from vllm.model_executor.layers.rejection_sampler import RejectionSampler
7 +from vllm.sequence import ExecuteModelRequest
8 +from vllm.spec_decode.metrics import AsyncMetricsCollector
9 +from vllm.spec_decode.multi_step_worker import MultiStepWorker
10 +from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker
11 +from vllm.spec_decode.top1_proposer import Top1Proposer
12 +
13 +from .utils import create_batch, mock_worker
14 +
15 +
16 +@pytest.mark.parametrize('queue_size', [2, 4])
17 +@pytest.mark.parametrize('batch_size', [1, 2, 3, 6])
18 +@pytest.mark.parametrize('k', [1, 2, 5, 7, 10])
19 +@torch.inference_mode()
20 +def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int):
21 + """Verify that speculative tokens are disabled when the batch size
22 + exceeds the threshold.
23 + """
24 + disable_by_batch_size = 3
25 +
26 + draft_worker = mock_worker(cls=MultiStepWorker)
27 + target_worker = mock_worker()
28 + rejection_sampler = MagicMock(spec=RejectionSampler)
29 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
30 + worker = SpecDecodeWorker(proposer_worker=draft_worker,
31 + scorer_worker=target_worker,
32 + rejection_sampler=rejection_sampler,
33 + metrics_collector=metrics_collector,
34 + disable_by_batch_size=disable_by_batch_size)
35 +
36 + exception_secret = 'artificial stop'
37 + draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
38 +
39 + seq_group_metadata_list, _, _ = create_batch(batch_size, k)
40 + execute_model_req = ExecuteModelRequest(
41 + seq_group_metadata_list=seq_group_metadata_list,
42 + num_lookahead_slots=k,
43 + running_queue_size=queue_size)
44 +
45 + with pytest.raises(ValueError, match=exception_secret):
46 + worker.execute_model(execute_model_req=execute_model_req)
47 +
48 + # When the batch size is larger than the threshold,
49 + # we expect no speculative tokens (0).
50 + expected_num_spec_tokens = None if queue_size < disable_by_batch_size else 0
51 + assert seq_group_metadata_list[
52 + 0].num_speculative_tokens == expected_num_spec_tokens
53 +
54 + draft_worker.sampler_output.side_effect = ValueError(exception_secret)
55 +
56 + proposer = Top1Proposer(
57 + worker=draft_worker,
58 + device='cpu', # not used
59 + vocab_size=100, # not used
60 + # Must be long enough to avoid being skipped due to length.
61 + max_proposal_len=1024,
62 + )
63 +
64 + if queue_size < disable_by_batch_size:
65 + # Should raise exception when executing the mocked draft model.
66 + with pytest.raises(ValueError, match=exception_secret):
67 + proposer.get_proposals(execute_model_req=ExecuteModelRequest(
68 + seq_group_metadata_list=seq_group_metadata_list,
69 + num_lookahead_slots=k), )
70 + else:
71 + # Should not execute the draft model because spec decode is disabled
72 + # for all requests. Accordingly, the proposal length should be 0.
73 + proposals = proposer.get_proposals(
74 + execute_model_req=ExecuteModelRequest(
75 + seq_group_metadata_list=seq_group_metadata_list,
76 + num_lookahead_slots=k), )
77 + assert proposals.proposal_lens.tolist() == [0] * batch_size
vllm/config.py
+24 -5
@@ -692,6 +692,7 @@ class SpeculativeConfig:
692 speculative_max_model_len: Optional[int],
693 enable_chunked_prefill: bool,
694 use_v2_block_manager: bool,
695 + speculative_disable_by_batch_size: Optional[int],
696 ngram_prompt_lookup_max: Optional[int],
697 ngram_prompt_lookup_min: Optional[int],
698 ) -> Optional["SpeculativeConfig"]:
@@ -720,6 +721,9 @@ class SpeculativeConfig:
721 use_v2_block_manager (bool): Whether vLLM is configured to use the
722 v2 block manager or not. Used for raising an error since the v2
723 block manager is required with spec decode.
724 + speculative_disable_by_batch_size (Optional[int]): Disable
725 + speculative decoding for new incoming requests when the number
726 + of enqueue requests is larger than this value, if provided.
727 ngram_prompt_lookup_max (Optional[int]): Max size of ngram token
728 window, if provided.
729 ngram_prompt_lookup_min (Optional[int]): Min size of ngram token
@@ -730,7 +734,7 @@ class SpeculativeConfig:
734 the necessary conditions are met, else None.
735 """
736
733 - if (speculative_model is None and num_speculative_tokens is None):
737 + if speculative_model is None and num_speculative_tokens is None:
738 return None
739
740 if speculative_model is not None and num_speculative_tokens is None:
@@ -739,6 +743,12 @@ class SpeculativeConfig:
743 "num_speculative_tokens to be provided, but found "
744 f"{speculative_model=} and {num_speculative_tokens=}.")
745
746 + if (speculative_disable_by_batch_size is not None
747 + and speculative_disable_by_batch_size < 2):
748 + raise ValueError("Expect the batch size threshold of disabling "
749 + "speculative decoding is > 1, but got "
750 + f"{speculative_disable_by_batch_size=}")
751 +
752 assert (speculative_model is not None
753 and num_speculative_tokens is not None)
754
@@ -807,6 +817,7 @@ class SpeculativeConfig:
817 draft_model_config,
818 draft_parallel_config,
819 num_speculative_tokens,
820 + speculative_disable_by_batch_size,
821 ngram_prompt_lookup_max,
822 ngram_prompt_lookup_min,
823 )
@@ -876,8 +887,9 @@ class SpeculativeConfig:
887 draft_model_config: ModelConfig,
888 draft_parallel_config: ParallelConfig,
889 num_speculative_tokens: int,
879 - ngram_prompt_lookup_max: int,
880 - ngram_prompt_lookup_min: int,
890 + speculative_disable_by_batch_size: Optional[int],
891 + ngram_prompt_lookup_max: Optional[int],
892 + ngram_prompt_lookup_min: Optional[int],
893 ):
894 """Create a SpeculativeConfig object.
895
@@ -886,12 +898,19 @@ class SpeculativeConfig:
898 draft_parallel_config: ParallelConfig for the draft model.
899 num_speculative_tokens: The number of tokens to sample from the
900 draft model before scoring with the target model.
901 + speculative_disable_by_batch_size: Disable speculative
902 + decoding for new incoming requests when the number of
903 + enqueue requests is larger than this value.
904 + ngram_prompt_lookup_max: Max size of ngram token window.
905 + ngram_prompt_lookup_min: Min size of ngram token window.
906 """
907 self.draft_model_config = draft_model_config
908 self.draft_parallel_config = draft_parallel_config
909 self.num_speculative_tokens = num_speculative_tokens
893 - self.ngram_prompt_lookup_max = ngram_prompt_lookup_max
894 - self.ngram_prompt_lookup_min = ngram_prompt_lookup_min
910 + self.speculative_disable_by_batch_size = \
911 + speculative_disable_by_batch_size
912 + self.ngram_prompt_lookup_max = ngram_prompt_lookup_max or 0
913 + self.ngram_prompt_lookup_min = ngram_prompt_lookup_min or 0
914
915 self._verify_args()
916
vllm/engine/arg_utils.py
+10
@@ -83,6 +83,7 @@ class EngineArgs:
83 speculative_model: Optional[str] = None
84 num_speculative_tokens: Optional[int] = None
85 speculative_max_model_len: Optional[int] = None
86 + speculative_disable_by_batch_size: Optional[int] = None
87 ngram_prompt_lookup_max: Optional[int] = None
88 ngram_prompt_lookup_min: Optional[int] = None
89
@@ -467,6 +468,13 @@ class EngineArgs:
468 'draft model. Sequences over this length will skip '
469 'speculation.')
470
471 + parser.add_argument(
472 + '--speculative-disable-by-batch-size',
473 + type=int,
474 + default=EngineArgs.speculative_disable_by_batch_size,
475 + help='Disable speculative decoding for new incoming requests '
476 + 'if the number of enqueue requests is larger than this value.')
477 +
478 parser.add_argument(
479 '--ngram-prompt-lookup-max',
480 type=int,
@@ -547,6 +555,8 @@ class EngineArgs:
555 target_dtype=self.dtype,
556 speculative_model=self.speculative_model,
557 num_speculative_tokens=self.num_speculative_tokens,
558 + speculative_disable_by_batch_size=self.
559 + speculative_disable_by_batch_size,
560 speculative_max_model_len=self.speculative_max_model_len,
561 enable_chunked_prefill=self.enable_chunked_prefill,
562 use_v2_block_manager=self.use_v2_block_manager,
vllm/executor/gpu_executor.py
+2
@@ -93,6 +93,8 @@ class GPUExecutor(ExecutorBase):
93 spec_decode_worker = SpecDecodeWorker.create_worker(
94 scorer_worker=target_worker,
95 draft_worker_kwargs=draft_worker_kwargs,
96 + disable_by_batch_size=self.speculative_config.
97 + speculative_disable_by_batch_size,
98 )
99
100 assert self.parallel_config.world_size == 1, (
vllm/model_executor/layers/rejection_sampler.py
+9 -2
@@ -12,15 +12,21 @@ class RejectionSampler(nn.Module):
12 https://arxiv.org/pdf/2302.01318.pdf.
13 """
14
15 - def __init__(self, strict_mode: bool = False):
15 + def __init__(self,
16 + disable_bonus_tokens: bool = True,
17 + strict_mode: bool = False):
18 """Create a rejection sampler.
19
20 Args:
21 + disable_bonus_tokens: Whether or not to disable the bonus token.
22 + Require when bonus tokens will cause corrupt KV cache for
23 + proposal methods that require KV cache.
24 strict_mode: Whether or not to perform shape/device/dtype checks
25 during sampling. This catches correctness issues but adds
26 nontrivial latency.
27 """
28 super().__init__()
29 + self._disable_bonus_tokens = disable_bonus_tokens
30 self._strict_mode = strict_mode
31
32 # NOTE: A "bonus token" is accepted iff all proposal tokens are
@@ -312,7 +318,8 @@ class RejectionSampler(nn.Module):
318 # proposal methods that require KV cache. We can fix it by "prefilling"
319 # the bonus token in the proposer. The following issue tracks the fix.
320 # https://github.com/vllm-project/vllm/issues/4212
315 - output_with_bonus_tokens[:, -1] = -1
321 + if self._disable_bonus_tokens:
322 + output_with_bonus_tokens[:, -1] = -1
323
324 # Fill the recovered token ids.
325 output.mul_(~after_false_mask).add_(
vllm/sequence.py
+6
@@ -612,6 +612,12 @@ class SequenceGroupMetadata:
612 self._token_chunk_size = token_chunk_size
613 self.do_sample = do_sample
614
615 + # The number of speculative tokens adopted in this request.
616 + # None means specuative decoding is not used.
617 + # Zero means speculative decoding is disabled for some reasons.
618 + # TODO: We should maintain this states out of the sequence group.
619 + self.num_speculative_tokens = None
620 +
621 if self._token_chunk_size is None:
622 if is_prompt:
623 self._token_chunk_size = list(seq_data.values())[0].get_len()
vllm/spec_decode/spec_decode_worker.py
+38 -21
@@ -1,5 +1,5 @@
1 from functools import cached_property
2 -from typing import List, Optional, Tuple
2 +from typing import Any, Dict, List, Optional, Tuple
3
4 import torch
5
@@ -54,7 +54,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
54 def create_worker(
55 cls,
56 scorer_worker: WorkerBase,
57 - draft_worker_kwargs,
57 + draft_worker_kwargs: Dict[str, Any],
58 + disable_by_batch_size: Optional[int],
59 ) -> "SpecDecodeWorker":
60
61 ngram_prompt_lookup_max = (
@@ -62,7 +63,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
63 ngram_prompt_lookup_min = (
64 draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
65
66 + disable_bonus_tokens = True
67 if ngram_prompt_lookup_max > 0:
68 + disable_bonus_tokens = False
69 proposer_worker = NGramWorker(**draft_worker_kwargs)
70 proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
71 ngram_prompt_lookup_max)
@@ -75,9 +78,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
78 return SpecDecodeWorker(
79 proposer_worker,
80 scorer_worker,
78 - # TODO(cade) disable strict mode for speedup.
79 - rejection_sampler=RejectionSampler(strict_mode=True),
80 - )
81 + disable_by_batch_size=disable_by_batch_size,
82 + rejection_sampler=RejectionSampler(
83 + disable_bonus_tokens=disable_bonus_tokens, ))
84
85 def __init__(
86 self,
@@ -85,6 +88,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
88 scorer_worker: WorkerBase,
89 rejection_sampler: RejectionSampler,
90 metrics_collector: Optional[AsyncMetricsCollector] = None,
91 + disable_by_batch_size: Optional[int] = None,
92 ):
93 """
94 Create a SpecDecodeWorker.
@@ -97,11 +101,14 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
101 Worker.
102 rejection_sampler: A Torch module used to perform modified rejection
103 sampling for speculative decoding.
104 + disable_by_batch_size: If the batch size is larger than this,
105 + disable speculative decoding for new incoming requests.
106 metrics_collector: Helper class for collecting metrics; can be set
107 for testing purposes.
108 """
109 self.proposer_worker = proposer_worker
110 self.scorer_worker = scorer_worker
111 + self.disable_by_batch_size = disable_by_batch_size or float("inf")
112 self.rejection_sampler = rejection_sampler
113
114 self._metrics = AsyncMetricsCollector(
@@ -199,27 +206,41 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
206 "speculative decoding "
207 "requires non-None seq_group_metadata_list")
208
209 + # When the batch size is too large, disable speculative decoding
210 + # to stop trading off throughput for latency.
211 + disable_all = (execute_model_req.running_queue_size >=
212 + self.disable_by_batch_size)
213 + if disable_all:
214 + for seq_group_metadata in execute_model_req.seq_group_metadata_list:
215 + # Once num_speculative_tokens is set to 0, the spec decode
216 + # of this request will be disabled forever.
217 + # TODO(comaniac): We currently store spec decoding specific
218 + # state in the global data structure, but we should maintain
219 + # this state within spec decode worker.
220 + seq_group_metadata.num_speculative_tokens = 0
221 +
222 # If no spec tokens, call the proposer and scorer workers normally.
203 - # Used for prefill.
223 + # This happens for prefill, or when the spec decode is disabled
224 + # for this batch.
225 if execute_model_req.num_lookahead_slots == 0 or len(
226 execute_model_req.seq_group_metadata_list) == 0:
206 - return self._run_no_spec(execute_model_req)
227 + return self._run_no_spec(execute_model_req,
228 + skip_proposer=disable_all)
229
230 return self._run_speculative_decoding_step(execute_model_req)
231
232 @nvtx_range("spec_decode_worker._run_no_spec")
211 - def _run_no_spec(
212 - self,
213 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
214 - """Run a prefill step, without any speculation. The input is sent to the
215 - proposer and scorer model so that the KV cache is consistent between the
216 - two.
233 + def _run_no_spec(self, execute_model_req: ExecuteModelRequest,
234 + skip_proposer: bool) -> List[SamplerOutput]:
235 + """Run a prefill step, without any speculation. The input is sent to
236 + the proposer and scorer model so that the KV cache is consistent
237 + between the two. When skip_proposer is True, the proposer model is
238 + not called, meaning that the kv-cache in proposer for requests is not
239 + updated, so they cannot enable spec decode in the rest decoding.
240 """
218 - #logger.info("run proposer worker no spec")
219 -
220 - self.proposer_worker.execute_model(execute_model_req)
241 + if not skip_proposer:
242 + self.proposer_worker.execute_model(execute_model_req)
243
222 - #logger.info("run target worker no spec")
244 sampler_output = self.scorer_worker.execute_model(execute_model_req)
245 assert len(sampler_output) == 1
246 sampler_output = sampler_output[0]
@@ -244,22 +265,18 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
265 sequence.
266 """
267
247 - #logger.info("get spec proposals")
268 # Generate proposals using draft worker.
269 proposals = self.proposer_worker.get_spec_proposals(execute_model_req)
270
251 - #logger.info("score proposals")
271 proposal_scores = self.scorer.score_proposals(
272 execute_model_req,
273 proposals,
274 )
275
257 - #logger.info("verify proposals")
276 accepted_token_ids, target_logprobs = self._verify_tokens(
277 execute_model_req.seq_group_metadata_list, proposal_scores,
278 proposals, execute_model_req.num_lookahead_slots)
279
262 - #logger.info("create output list")
280 return self._create_output_sampler_list(
281 execute_model_req.seq_group_metadata_list,
282 accepted_token_ids,
vllm/spec_decode/top1_proposer.py
+17 -6
@@ -56,7 +56,7 @@ class Top1Proposer(SpeculativeProposer):
56 proposal_lens,
57 nonzero_proposal_len_seqs,
58 nonzero_proposal_len_indices,
59 - ) = self._split_by_max_model_len(seq_group_metadata_list, proposal_len)
59 + ) = self._split_by_proposal_len(seq_group_metadata_list, proposal_len)
60
61 if nonzero_proposal_len_seqs:
62 # Speculate tokens using the draft worker for the speculative
@@ -97,17 +97,27 @@ class Top1Proposer(SpeculativeProposer):
97
98 return proposals
99
100 - def _split_by_max_model_len(
100 + def _split_by_proposal_len(
101 self,
102 seq_group_metadata_list: List[SequenceGroupMetadata],
103 proposal_len: int,
104 ) -> Tuple[List[int], List[SequenceGroupMetadata], List[int]]:
105 - """Determine which sequences would exceed the max model length."""
105 + """Split sequences by two groups:
106 + 1. Sequences with non-zero proposal length.
107 + 2. Sequences with zero proposal length (due to disabled speculation
108 + or exceed the maximum model length).
109 + """
110
111 proposal_lens: List[int] = []
112 nonzero_proposal_len_seqs: List[SequenceGroupMetadata] = []
113 nonzero_proposal_len_indices: List[int] = []
114 for i, seq_group_metadata in enumerate(seq_group_metadata_list):
115 + # The speculative decoding for this request has been disabled
116 + # (e.g. due to high traffic).
117 + if seq_group_metadata.num_speculative_tokens == 0:
118 + proposal_lens.append(0)
119 + continue
120 +
121 seq_data = next(iter(seq_group_metadata.seq_data.values()))
122 seq_len = seq_data.get_len()
123
@@ -115,13 +125,14 @@ class Top1Proposer(SpeculativeProposer):
125 # are supported.
126 # If max_proposal_len is defined, then we shall no exccess this
127 # quota for nonzero_proposal
128 + new_k = 0
129 if (self.max_proposal_len is None
130 or seq_len + proposal_len < self.max_proposal_len):
120 - proposal_lens.append(proposal_len)
131 + new_k = proposal_len
132 nonzero_proposal_len_seqs.append(seq_group_metadata)
133 nonzero_proposal_len_indices.append(i)
123 - else:
124 - proposal_lens.append(0)
134 + proposal_lens.append(new_k)
135 + seq_group_metadata.num_speculative_tokens = new_k
136
137 return (
138 proposal_lens,