@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 603ad84815
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Refactoring sampler and support prompt logprob for chunked prefill (#4309)

SangBin Cho committed Apr 26, 2024 at 22:02 UTC 603ad8481594321ceae7d54e2c0050b3638c6502
18 files changed +862 -633
tests/samplers/test_logprobs.py
+41 -3
@@ -9,15 +9,26 @@ MODELS = ["facebook/opt-125m"]
9
10 @pytest.mark.parametrize("model", MODELS)
11 @pytest.mark.parametrize("dtype", ["half"])
12 +@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
13 +@pytest.mark.parametrize("num_top_logprobs", [6]) # 32000 == vocab_size
14 def test_get_prompt_logprobs(
15 hf_runner,
16 vllm_runner,
17 model,
18 dtype,
19 + chunked_prefill_token_size: int,
20 + num_top_logprobs: int,
21 example_prompts,
22 ):
23 + max_num_seqs = 256
24 + enable_chunked_prefill = False
25 + max_num_batched_tokens = None
26 + if chunked_prefill_token_size != -1:
27 + enable_chunked_prefill = True
28 + max_num_seqs = min(chunked_prefill_token_size, max_num_seqs)
29 + max_num_batched_tokens = chunked_prefill_token_size
30 +
31 max_tokens = 5
20 - num_top_logprobs = 6
32 hf_model = hf_runner(model, dtype=dtype)
33 hf_logprobs = hf_model.generate_greedy_logprobs(
34 example_prompts,
@@ -25,10 +36,17 @@ def test_get_prompt_logprobs(
36 )
37 del hf_model
38
28 - vllm_model = vllm_runner(model, dtype=dtype, max_logprobs=num_top_logprobs)
39 + vllm_model = vllm_runner(
40 + model,
41 + dtype=dtype,
42 + max_logprobs=num_top_logprobs,
43 + enable_chunked_prefill=enable_chunked_prefill,
44 + max_num_batched_tokens=max_num_batched_tokens,
45 + max_num_seqs=max_num_seqs,
46 + )
47 vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
48 logprobs=num_top_logprobs,
31 - prompt_logprobs=5,
49 + prompt_logprobs=num_top_logprobs,
50 temperature=0.0)
51 vllm_results = vllm_model.model.generate(
52 example_prompts, sampling_params=vllm_sampling_params)
@@ -52,9 +70,18 @@ def test_get_prompt_logprobs(
70 "The output text from the top logprob for each token position "
71 "should be the same as the output text in the result.")
72
73 + # The first prompt logprob is always None
74 + assert result.prompt_logprobs[0] is None
75 + for prompt_logprobs in result.prompt_logprobs[1:]:
76 + # If the prompt token is not included in the top X
77 + # logprob, it can return 1 more data
78 + assert (len(prompt_logprobs) == num_top_logprobs
79 + or len(prompt_logprobs) == num_top_logprobs + 1)
80 +
81 # Test whether prompt logprobs are consistent with HF
82 for vllm_result, hf_logprob in zip(vllm_results, hf_logprobs):
83 # Check prompt logprobs
84 + # The first prompt logprob is always None, so we compare it from 1:.
85 vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
86 for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
87 for token_id, logprob in vllm_prompt_logprob_dict.items():
@@ -74,6 +101,17 @@ def test_get_prompt_logprobs(
101 "The token should be decoded by the time it is returned "
102 " to the user.")
103
104 + # Test if prompt logprobs are correctly set.
105 + for vllm_result in vllm_results:
106 + token_ids = vllm_result.prompt_token_ids
107 + prompt_logprobs = vllm_result.prompt_logprobs
108 +
109 + # The first token doesn't have logprob.
110 + assert prompt_logprobs[0] is None
111 +
112 + for token_id, logprob_dict in zip(token_ids[1:], prompt_logprobs[1:]):
113 + assert token_id in logprob_dict
114 +
115
116 def test_max_logprobs():
117 runner = VllmRunner("facebook/opt-125m", max_logprobs=1)
tests/samplers/test_sampler.py
+31 -16
@@ -8,6 +8,7 @@ import torch
8 from transformers import GenerationConfig, GenerationMixin
9
10 from vllm.model_executor.layers.sampler import Sampler
11 +from vllm.model_executor.sampling_metadata import SamplingMetadata
12 from vllm.model_executor.utils import set_random_seed
13 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
14 from vllm.utils import Counter
@@ -54,6 +55,7 @@ def _do_sample(
55 sampler: MockLogitsSampler,
56 model_runner: ModelRunner,
57 sampling_params: SamplingParams,
58 + device: str,
59 ):
60 seq_group_metadata_list = []
61 prompt_lens = []
@@ -68,9 +70,12 @@ def _do_sample(
70 ))
71 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
72
71 - sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
72 - prompt_lens,
73 - subquery_lens=prompt_lens)
73 + sampling_metadata = SamplingMetadata.prepare(
74 + seq_group_metadata_list,
75 + prompt_lens,
76 + subquery_lens=prompt_lens,
77 + device=device,
78 + pin_memory=model_runner.pin_memory)
79 return sampler(logits=input_tensor, sampling_metadata=sampling_metadata)
80
81
@@ -85,7 +90,7 @@ def test_sampler_all_greedy(seed: int, device: str):
90
91 sampling_params = SamplingParams(temperature=0)
92 sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
88 - sampling_params)
93 + sampling_params, device)
94 expected = torch.argmax(fake_logits, dim=-1)
95 for i, sequence_output in enumerate(sampler_output):
96 for nth_output in sequence_output.samples:
@@ -111,7 +116,7 @@ def test_sampler_all_random(seed: int, device: str):
116 n=random.randint(1, 10),
117 )
118 sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
114 - sampling_params)
119 + sampling_params, device)
120
121 for i, sequence_output in enumerate(sampler_output):
122 for nth_output in sequence_output.samples:
@@ -137,7 +142,7 @@ def test_sampler_all_random_seed(seed: int, device: str):
142 seed=random.randint(0, 10000),
143 )
144 sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
140 - sampling_params)
145 + sampling_params, device)
146
147 for i, sequence_output in enumerate(sampler_output):
148 for nth_output in sequence_output.samples:
@@ -160,10 +165,10 @@ def test_sampler_all_random_seed_deterministic(seed: int, device: str):
165 seed=random.randint(0, 10000),
166 )
167 first_sampler_output = _do_sample(batch_size, fake_logits, sampler,
163 - model_runner, sampling_params)
168 + model_runner, sampling_params, device)
169
170 second_sampler_output = _do_sample(batch_size, fake_logits, sampler,
166 - model_runner, sampling_params)
171 + model_runner, sampling_params, device)
172
173 assert first_sampler_output == second_sampler_output
174
@@ -183,7 +188,8 @@ def test_sampler_all_beam(seed: int, device: str):
188 best_of=2,
189 use_beam_search=True,
190 )
186 - _do_sample(batch_size, fake_logits, sampler, model_runner, sampling_params)
191 + _do_sample(batch_size, fake_logits, sampler, model_runner, sampling_params,
192 + device)
193 # no assertion here as I am not sure how to determine whether
194 # the outputs are expected - in other words, this just tests
195 # whether there are no exceptions in the sampler
@@ -443,10 +449,12 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
449 "batch size")
450
451 _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
446 - sampling_metadata = model_runner._prepare_sample(
452 + sampling_metadata = SamplingMetadata.prepare(
453 seq_group_metadata_list,
454 prompt_lens=prompt_lens if prompt_lens else None,
449 - subquery_lens=prompt_lens if prompt_lens else None)
455 + subquery_lens=prompt_lens if prompt_lens else None,
456 + device=device,
457 + pin_memory=model_runner.pin_memory)
458 # the logits tensor is modified in-place by the sampler
459 _ = sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
460
@@ -530,8 +538,12 @@ def test_sampler_mixed(seed: int, device: str):
538 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
539
540 def test_sampling(model_runner: ModelRunner):
533 - sampling_metadata = model_runner._prepare_sample(
534 - seq_group_metadata_list, prompt_lens, subquery_lens=prompt_lens)
541 + sampling_metadata = SamplingMetadata.prepare(
542 + seq_group_metadata_list,
543 + prompt_lens,
544 + subquery_lens=prompt_lens,
545 + device=device,
546 + pin_memory=model_runner.pin_memory)
547 sampler_output = sampler(logits=fake_logits,
548 sampling_metadata=sampling_metadata)
549
@@ -627,9 +639,12 @@ def test_sampler_top_k_top_p(seed: int, device: str):
639 ))
640 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
641
630 - sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
631 - prompt_lens,
632 - subquery_lens=prompt_lens)
642 + sampling_metadata = SamplingMetadata.prepare(
643 + seq_group_metadata_list,
644 + prompt_lens,
645 + subquery_lens=prompt_lens,
646 + device=device,
647 + pin_memory=model_runner.pin_memory)
648
649 sample_probs = None
650
tests/test_logits_processor.py
+7 -3
@@ -6,6 +6,7 @@ import pytest
6 import torch
7
8 from vllm.model_executor.layers.logits_processor import LogitsProcessor
9 +from vllm.model_executor.sampling_metadata import SamplingMetadata
10 from vllm.model_executor.utils import set_random_seed
11 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12 from vllm.worker.model_runner import ModelRunner
@@ -82,9 +83,12 @@ def test_logits_processors(seed: int, device: str):
83 ))
84 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
85
85 - sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
86 - prompt_lens,
87 - subquery_lens=prompt_lens)
86 + sampling_metadata = SamplingMetadata.prepare(
87 + seq_group_metadata_list,
88 + prompt_lens,
89 + subquery_lens=prompt_lens,
90 + device=model_runner.device,
91 + pin_memory=model_runner.pin_memory)
92 logits_processor_output = logits_processor(
93 embedding=None,
94 hidden_states=input_tensor,
tests/worker/test_model_runner.py
+13 -6
@@ -2,6 +2,7 @@ import pytest
2 import torch
3
4 from vllm.config import ModelConfig, SchedulerConfig
5 +from vllm.model_executor.sampling_metadata import SamplingMetadata
6 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
7 from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
8
@@ -97,9 +98,12 @@ def test_prepare_prompt(batch_size):
98 assert len(input_positions) == sum(prompt_lens)
99 torch.testing.assert_close(input_tokens, input_positions)
100
100 - sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
101 - prompt_lens,
102 - subquery_lens=prompt_lens)
101 + sampling_metadata = SamplingMetadata.prepare(
102 + seq_group_metadata_list,
103 + prompt_lens,
104 + subquery_lens=prompt_lens,
105 + device=model_runner.device,
106 + pin_memory=model_runner.pin_memory)
107 assert len(input_tokens) == sum(prompt_lens)
108 assert len(input_positions) == sum(prompt_lens)
109 actual = sampling_metadata.selected_token_indices
@@ -195,9 +199,12 @@ def test_prepare_decode_cuda_graph(batch_size):
199 for prompt_len in prompt_lens:
200 expected_selected_token_indices.append(selected_token_start_idx)
201 selected_token_start_idx += 1
198 - sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
199 - prompt_lens,
200 - subquery_lens=prompt_lens)
202 + sampling_metadata = SamplingMetadata.prepare(
203 + seq_group_metadata_list,
204 + prompt_lens,
205 + subquery_lens=prompt_lens,
206 + device=model_runner.device,
207 + pin_memory=model_runner.pin_memory)
208 actual = sampling_metadata.selected_token_indices
209 expected = torch.tensor(expected_selected_token_indices,
210 device=actual.device,
vllm/core/scheduler.py
+15
@@ -915,6 +915,20 @@ class Scheduler:
915 self.block_manager.get_common_computed_block_ids(
916 seq_group.get_seqs(status=SequenceStatus.RUNNING)))
917
918 + do_sample = True
919 + if seq_group.is_prefill():
920 + seqs = seq_group.get_seqs()
921 + # Prefill has only 1 sequence.
922 + assert len(seqs) == 1
923 + # In the next iteration, all prompt tokens are not computed.
924 + # It means the prefill is chunked, and we don't need sampling.
925 + # NOTE: We use get_len instead of get_prompt_len because when
926 + # a sequence is preempted, prefill includes previous generated
927 + # output tokens.
928 + if (token_chunk_size + seqs[0].data.get_num_computed_tokens() <
929 + seqs[0].data.get_len()):
930 + do_sample = False
931 +
932 # It assumes the scheduled_seq_groups is ordered by
933 # prefill < decoding.
934 is_prompt = seq_group.is_prefill()
@@ -924,6 +938,7 @@ class Scheduler:
938 seq_data=seq_data,
939 sampling_params=seq_group.sampling_params,
940 block_tables=block_tables,
941 + do_sample=do_sample,
942 token_chunk_size=token_chunk_size,
943 lora_request=seq_group.lora_request,
944 computed_block_nums=common_computed_block_nums,
vllm/engine/async_llm_engine.py
+1 -1
@@ -219,7 +219,7 @@ class _AsyncLLMEngine(LLMEngine):
219
220 request_outputs = self._process_model_outputs(
221 output, scheduler_outputs.scheduled_seq_groups,
222 - scheduler_outputs.ignored_seq_groups)
222 + scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
223
224 # Log stats.
225 if self.log_stats:
vllm/engine/llm_engine.py
+13 -12
@@ -22,7 +22,7 @@ from vllm.lora.request import LoRARequest
22 from vllm.outputs import RequestOutput
23 from vllm.sampling_params import SamplingParams
24 from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
25 - SequenceGroup, SequenceStage)
25 + SequenceGroup, SequenceGroupMetadata)
26 from vllm.transformers_utils.detokenizer import Detokenizer
27 from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
28 get_tokenizer_group)
@@ -476,9 +476,12 @@ class LLMEngine:
476 return self.scheduler.has_unfinished_seqs()
477
478 def _process_model_outputs(
479 - self, output: List[SamplerOutput],
480 - scheduled_seq_groups: List[SequenceGroup],
481 - ignored_seq_groups: List[SequenceGroup]) -> List[RequestOutput]:
479 + self,
480 + output: List[SamplerOutput],
481 + scheduled_seq_groups: List[SequenceGroup],
482 + ignored_seq_groups: List[SequenceGroup],
483 + seq_group_metadata_list: List[SequenceGroupMetadata],
484 + ) -> List[RequestOutput]:
485 """Apply the model output to the sequences in the scheduled seq groups.
486
487 Returns RequestOutputs that can be returned to the client.
@@ -492,17 +495,15 @@ class LLMEngine:
495 sampler_outputs=output, num_seq_groups=len(scheduled_seq_groups))
496
497 # Update the scheduled sequence groups with the model outputs.
495 - for scheduled_seq_group, outputs in zip(scheduled_seq_groups,
496 - output_by_sequence_group):
498 + for scheduled_seq_group, outputs, seq_group_meta in zip(
499 + scheduled_seq_groups, output_by_sequence_group,
500 + seq_group_metadata_list):
501 seq_group = scheduled_seq_group.seq_group
502 seq_group.update_num_computed_tokens(
503 scheduled_seq_group.token_chunk_size)
504
501 - # If all sequences in the sequence group are in DECODE, then we can
502 - # process the output tokens. Otherwise, they are (chunked) prefill
503 - # samples and should not be processed.
504 - stages = [seq.data._stage for seq in seq_group.seqs_dict.values()]
505 - if all(stage == SequenceStage.DECODE for stage in stages):
505 + self.output_processor.process_prompt_logprob(seq_group, outputs)
506 + if seq_group_meta.do_sample:
507 self.output_processor.process_outputs(seq_group, outputs)
508
509 # Free the finished sequence groups.
@@ -585,7 +586,7 @@ class LLMEngine:
586
587 request_outputs = self._process_model_outputs(
588 output, scheduler_outputs.scheduled_seq_groups,
588 - scheduler_outputs.ignored_seq_groups)
589 + scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
590
591 # Log stats.
592 if self.log_stats:
vllm/engine/output_processor/interfaces.py
+6
@@ -68,3 +68,9 @@ class SequenceGroupOutputProcessor(ABC):
68 scheduler.
69 """
70 pass
71 +
72 + @abstractmethod
73 + def process_prompt_logprob(self, seq_group: SequenceGroup,
74 + outputs: List[SequenceGroupOutput]) -> None:
75 + """Update prompt logprobs received from outputs to seq_group."""
76 + pass
vllm/engine/output_processor/multi_step.py
+9
@@ -44,6 +44,15 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
44 self.get_tokenizer_for_seq = get_tokenizer_for_seq
45 self.stop_checker = stop_checker
46
47 + def process_prompt_logprob(self, seq_group: SequenceGroup,
48 + outputs: List[SequenceGroupOutput]) -> None:
49 + # TODO(sang): Prompt logprob currently not implemented in multi step
50 + # workers.
51 + logger.warning(
52 + "Prompt logprob is not supported by multi step workers. "
53 + "(e.g., speculative decode uses multi step workers).")
54 + pass
55 +
56 def process_outputs(self, sequence_group: SequenceGroup,
57 outputs: List[SequenceGroupOutput]) -> None:
58 """Append new tokens in the outputs to sequences in the sequence group.
vllm/engine/output_processor/single_step.py
+14 -8
@@ -55,17 +55,23 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
55 ), f"{type(self)} does not support multiple outputs per step"
56 return self._process_sequence_group_outputs(sequence_group, outputs[0])
57
58 - def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
59 - outputs: SequenceGroupOutput) -> None:
60 -
61 - # Process prompt logprobs
62 - prompt_logprobs = outputs.prompt_logprobs
63 - if prompt_logprobs is not None and \
64 - seq_group.sampling_params.detokenize and self.detokenizer:
58 + def process_prompt_logprob(self, seq_group: SequenceGroup,
59 + outputs: List[SequenceGroupOutput]) -> None:
60 + assert len(outputs) == 1, ("Single step should only has 1 output.")
61 + output = outputs[0]
62 + prompt_logprobs = output.prompt_logprobs
63 + if (prompt_logprobs is not None
64 + and seq_group.sampling_params.detokenize and self.detokenizer):
65 self.detokenizer.decode_prompt_logprobs_inplace(
66 seq_group, prompt_logprobs)
67 - seq_group.prompt_logprobs = prompt_logprobs
67 + if not seq_group.prompt_logprobs:
68 + # The first prompt token's logprob is None because it doesn't
69 + # have tokens that are precedent.
70 + seq_group.prompt_logprobs = [None]
71 + seq_group.prompt_logprobs.extend(prompt_logprobs)
72
73 + def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
74 + outputs: SequenceGroupOutput) -> None:
75 # Process samples
76 samples = outputs.samples
77 parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
vllm/engine/output_processor/util.py
+4 -3
@@ -1,10 +1,11 @@
1 from typing import List
2
3 -from vllm.sequence import SamplerOutput
3 +from vllm.sequence import SamplerOutput, SequenceGroupOutput
4
5
6 -def create_output_by_sequence_group(sampler_outputs: List[SamplerOutput],
7 - num_seq_groups: int):
6 +def create_output_by_sequence_group(
7 + sampler_outputs: List[SamplerOutput],
8 + num_seq_groups: int) -> List[List[SequenceGroupOutput]]:
9 """Helper method which transforms a 2d list organized by
10 [step][sequence group] into [sequence group][step].
11 """
vllm/model_executor/layers/logits_processor.py
+12 -15
@@ -83,30 +83,27 @@ def _apply_logits_processors(
83 logits: torch.Tensor,
84 sampling_metadata: SamplingMetadata,
85 ) -> torch.Tensor:
86 - logits_row_idx = 0
86 found_logits_processors = False
88 - for i, seq_group in enumerate(sampling_metadata.seq_groups):
89 - seq_ids, sampling_params = seq_group
87 + logits_processed = 0
88 + for seq_group in sampling_metadata.seq_groups:
89 + seq_ids = seq_group.seq_ids
90 + sampling_params = seq_group.sampling_params
91 logits_processors = sampling_params.logits_processors
91 - # handle prompt_logprobs by skipping rows in logits added for
92 - # the prompt tokens (prompt logprobs are not processed)
93 - if (i < sampling_metadata.num_prompts
94 - and sampling_params.prompt_logprobs is not None):
95 - assert len(seq_ids) == 1
96 - logits_row_idx += sampling_metadata.prompt_lens[i] - 1
92
93 if logits_processors:
94 found_logits_processors = True
100 - for seq_id in seq_ids:
95 + for seq_id, logits_row_idx in zip(seq_ids,
96 + seq_group.sample_indices):
97 logits_row = logits[logits_row_idx]
102 - token_ids = sampling_metadata.seq_data[seq_id].output_token_ids
98 + token_ids = seq_group.seq_data[seq_id].output_token_ids
99 for logits_processor in logits_processors:
100 logits_row = logits_processor(token_ids, logits_row)
101 logits[logits_row_idx] = logits_row
106 - logits_row_idx += 1
107 - else:
108 - logits_row_idx += len(seq_ids)
102 +
103 + logits_processed += len(seq_group.sample_indices) + len(
104 + seq_group.prompt_logprob_indices)
105 +
106 if found_logits_processors:
107 # verifies that no rows in logits were missed unexpectedly
111 - assert logits_row_idx == logits.shape[0]
108 + assert logits_processed == logits.shape[0]
109 return logits
vllm/model_executor/layers/sampler.py
+366 -178
@@ -7,11 +7,11 @@ import torch.nn as nn
7
8 from vllm.model_executor.layers.ops.sample import sample as sample_triton
9 from vllm.model_executor.sampling_metadata import (SamplingMetadata,
10 - SamplingTensors)
11 -from vllm.sampling_params import SamplingParams, SamplingType
10 + SamplingTensors,
11 + SequenceGroupToSample)
12 +from vllm.sampling_params import SamplingType
13 from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
13 - SamplerOutput, SequenceData, SequenceGroupOutput,
14 - SequenceOutput)
14 + SamplerOutput, SequenceGroupOutput, SequenceOutput)
15
16
17 class Sampler(nn.Module):
@@ -48,11 +48,14 @@ class Sampler(nn.Module):
48 logits: torch.Tensor,
49 sampling_metadata: SamplingMetadata,
50 ) -> Optional[SamplerOutput]:
51 + """
52 + Args:
53 + logits: (num_tokens, vocab_size).
54 + sampling_metadata: Metadata for sampling.
55 + """
56 assert logits is not None
57 _, vocab_size = logits.shape
58
54 - # Apply min_tokens penalty which sets stop tokens to -inf if min_tokens
55 - # have not been generated yet
59 logits = _apply_min_tokens_penalty(logits, sampling_metadata)
60
61 # Prepare sampling tensors with pinned memory to avoid blocking.
@@ -83,7 +86,6 @@ class Sampler(nn.Module):
86 # Compute the probabilities.
87 probs = torch.softmax(logits, dim=-1, dtype=torch.float)
88 # Compute the log probabilities.
86 - # Use log_softmax to ensure numerical stability.
89 logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
90
91 # Sample the next tokens.
@@ -149,24 +151,28 @@ def _apply_min_tokens_penalty(
151 logits: torch.Tensor,
152 sampling_metadata: SamplingMetadata,
153 ) -> torch.Tensor:
154 + """Apply min_tokens penalty which sets stop tokens to -inf if min_tokens
155 + have not been generated yet
156 + """
157 # list of indices in logits that will be set to -inf
158 logits_to_penalize = []
154 - start_idx = 0
155 - for i, seq_group in enumerate(sampling_metadata.seq_groups):
156 - seq_ids, sampling_params = seq_group
157 -
158 - # handle prompt_logprobs by skipping rows in logits added for the prompt
159 - # tokens (prompt logprobs are not penalized)
160 - if (i < sampling_metadata.num_prompts
161 - and sampling_params.prompt_logprobs is not None):
162 - assert len(seq_ids) == 1
163 - start_idx += sampling_metadata.prompt_lens[i] - 1
159 + logits_applied = 0
160 + for seq_group in sampling_metadata.seq_groups:
161 + seq_ids = seq_group.seq_ids
162 + sampling_params = seq_group.sampling_params
163 +
164 + sample_indices = seq_group.sample_indices
165 + logits_applied += len(sample_indices) + len(
166 + seq_group.prompt_logprob_indices)
167 + if not seq_group.do_sample:
168 + continue
169
170 + start_idx = sample_indices[0]
171 min_tokens = sampling_params.min_tokens
172 if min_tokens > 0:
173 seqs_to_penalize = []
174 for i, seq_id in enumerate(seq_ids):
169 - seq_data = sampling_metadata.seq_data[seq_id]
175 + seq_data = seq_group.seq_data[seq_id]
176 if len(seq_data.output_token_ids) < min_tokens:
177 seqs_to_penalize.append(i)
178
@@ -180,15 +186,13 @@ def _apply_min_tokens_penalty(
186 logits_to_penalize.extend(
187 itertools.product(seqs_to_penalize, token_ids_to_penalize))
188
183 - start_idx += len(seq_ids)
184 -
189 if logits_to_penalize:
190 # use zip and * to group indices along each dimension
191 # eg. [ (1,2), (1,3), (5,6) ] -> ( (1,1,5), (2,3,6) )
192 logits[tuple(zip(*logits_to_penalize))] = -float("inf")
193
194 # verifies that no rows in logits were missed unexpectedly
191 - assert start_idx == logits.shape[0]
195 + assert logits_applied == logits.shape[0]
196 return logits
197
198
@@ -265,14 +269,30 @@ def _apply_min_p(
269
270
271 def _greedy_sample(
268 - selected_seq_groups: List[Tuple[List[int], SamplingParams]],
272 + selected_seq_groups: List[SequenceGroupToSample],
273 samples: torch.Tensor,
274 ) -> List[Tuple[List[int], List[int]]]:
275 + """Run greedy sampling on a given samples.
276 +
277 + Args:
278 + selected_seq_groups: A list of sequence groups batched.
279 + samples: (num_selected_samples,) A tensor of samples. The length of
280 + samples could be smaller than selected_seq_groups if
281 + seq_group.do_sample is False.
282 + Returns:
283 + Tuple of (next_token_ids, parent_ids). The length of returned list is
284 + same as the length of selected_seq_groups. If the corresponding
285 + seq_group has do_sample=False, tuple contains ([], [])
286 + """
287 samples = samples.tolist()
288 sample_idx = 0
289 results = []
290 for seq_group in selected_seq_groups:
275 - seq_ids, _ = seq_group
291 + if not seq_group.do_sample:
292 + results.append(([], []))
293 + continue
294 +
295 + seq_ids = seq_group.seq_ids
296 num_parent_seqs = len(seq_ids)
297 assert num_parent_seqs == 1, (
298 "Greedy sampling should have only one seq.")
@@ -284,16 +304,33 @@ def _greedy_sample(
304
305
306 def _random_sample(
287 - selected_seq_groups: List[Tuple[List[int], SamplingParams]],
288 - is_prompts: List[bool],
307 + selected_seq_groups: List[SequenceGroupToSample],
308 random_samples: torch.Tensor,
309 ) -> List[Tuple[List[int], List[int]]]:
310 + """Run random sampling on a given samples.
311 +
312 + Args:
313 + selected_seq_groups: A list of sequence groups batched.
314 + random_samples: (num_selected_samples,) A tensor of samples. The
315 + length of samples could be smaller than selected_seq_groups if
316 + seq_group.do_sample is False.
317 + Returns:
318 + Tuple of (next_token_ids, parent_ids). The length of returned list is
319 + same as the length of selected_seq_groups. If the corresponding
320 + seq_group has do_sample=False, tuple contains ([], [])
321 + """
322 # Find the maximum best_of value of the prompt phase requests.
323 random_samples = random_samples.cpu()
324 sample_idx = 0
325 results = []
295 - for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
296 - seq_ids, sampling_params = seq_group
326 + for seq_group in selected_seq_groups:
327 + if not seq_group.do_sample:
328 + results.append(([], []))
329 + continue
330 +
331 + seq_ids = seq_group.seq_ids
332 + sampling_params = seq_group.sampling_params
333 + is_prompt = seq_group.is_prompt
334 num_parent_seqs = len(seq_ids)
335 if is_prompt:
336 # Prompt phase.
@@ -311,11 +348,20 @@ def _random_sample(
348
349
350 def _beam_search_sample(
314 - selected_seq_groups: List[Tuple[List[int], SamplingParams]],
315 - is_prompts: List[bool],
316 - seq_data: Dict[int, SequenceData],
351 + selected_seq_groups: List[SequenceGroupToSample],
352 logprobs: torch.Tensor,
353 ) -> List[Tuple[List[int], List[int]]]:
354 + """Run beam sampling on a given samples.
355 +
356 + Args:
357 + selected_seq_groups: A list of sequence groups batched.
358 + logprobs: (num_selected_samples, vocab_size,) A tensor of logprob
359 + on selected sample indices.
360 + Returns:
361 + Tuple of (next_token_ids, parent_ids). The length of returned list is
362 + same as the length of selected_seq_groups. If the corresponding
363 + seq_group has do_sample=False, tuple contains ([], [])
364 + """
365 # We sample 2 * beam_width candidates to make sure that with high
366 # probability we can get `beam_width` candidates in addition to
367 # the finished sequences for the next iteration. See
@@ -327,8 +373,13 @@ def _beam_search_sample(
373 # other sampling methods.
374 sample_idx = 0
375 results = []
330 - for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
331 - seq_ids, sampling_params = seq_group
376 + for seq_group in selected_seq_groups:
377 + if not seq_group.do_sample:
378 + results.append(([], []))
379 + continue
380 +
381 + is_prompt = seq_group.is_prompt
382 + seq_ids, sampling_params = seq_group.seq_ids, seq_group.sampling_params
383 num_parent_seqs = len(seq_ids)
384 beam_width = sampling_params.best_of
385 seq_group_logprobs = logprobs[sample_idx:sample_idx + num_parent_seqs]
@@ -343,7 +394,8 @@ def _beam_search_sample(
394 else:
395 # Generation phase.
396 cumulative_logprobs = [
346 - seq_data[seq_id].cumulative_logprob for seq_id in seq_ids
397 + seq_group.seq_data[seq_id].cumulative_logprob
398 + for seq_id in seq_ids
399 ]
400 cumulative_logprobs = torch.tensor(
401 cumulative_logprobs,
@@ -371,8 +423,7 @@ def _beam_search_sample(
423 def _multinomial(
424 probs: torch.Tensor,
425 num_samples: int,
374 - seq_groups: Optional[List[Tuple[List[int], SamplingParams]]] = None,
375 - generators: Optional[List[torch.Generator]] = None,
426 + seq_groups: Optional[List[SequenceGroupToSample]] = None,
427 ) -> torch.Tensor:
428 if num_samples > 1:
429 # This is equivalent to torch.repeat_interleaved (which also
@@ -388,9 +439,11 @@ def _multinomial(
439 q.exponential_()
440 else:
441 sample_idx = 0
391 - for (seq_ids, _), generator in zip(seq_groups, generators):
442 + for seq_group in seq_groups:
443 + seq_ids = seq_group.seq_ids
444 next_sample_idx = sample_idx + len(seq_ids) * num_samples
393 - q[sample_idx:next_sample_idx].exponential_(generator=generator)
445 + q[sample_idx:next_sample_idx].exponential_(
446 + generator=seq_group.generator)
447 sample_idx = next_sample_idx
448 return probs.div_(q).argmax(dim=1).view(-1, num_samples)
449
@@ -405,7 +458,7 @@ def _sample_with_torch(
458 categorized_seq_group_ids = {t: [] for t in SamplingType}
459 categorized_sample_indices = sampling_metadata.categorized_sample_indices
460 for i, seq_group in enumerate(sampling_metadata.seq_groups):
408 - _, sampling_params = seq_group
461 + sampling_params = seq_group.sampling_params
462 sampling_type = sampling_params.sampling_type
463 categorized_seq_group_ids[sampling_type].append(i)
464
@@ -429,13 +482,11 @@ def _sample_with_torch(
482 num_tokens = len(sample_indices)
483 if num_tokens == 0:
484 continue
432 - seq_group_ids = categorized_seq_group_ids[sampling_type]
433 - seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
434 - is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
435 - sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
436 - is_prompts, sample_indices)
437 - long_sample_indices = sample_indices.long()
485
486 + seq_group_id = categorized_seq_group_ids[sampling_type]
487 + seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_id]
488 + sample_metadata[sampling_type] = (seq_group_id, seq_groups)
489 + long_sample_indices = sample_indices.long()
490 if sampling_type == SamplingType.GREEDY:
491 greedy_samples = torch.argmax(logprobs[long_sample_indices],
492 dim=-1)
@@ -455,14 +506,13 @@ def _sample_with_torch(
506
507 elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
508 max_best_of_in_batch = 1
458 - for seq_group, is_prompt in zip(seq_groups, is_prompts):
459 - if is_prompt:
460 - _, sampling_params = seq_group
509 + for seq_group in seq_groups:
510 + if seq_group.is_prompt:
511 + sampling_params = seq_group.sampling_params
512 max_best_of_in_batch = max(max_best_of_in_batch,
513 sampling_params.best_of)
514 seeded_args = {} if sampling_type == SamplingType.RANDOM else {
515 "seq_groups": seq_groups,
465 - "generators": sampling_metadata.generators,
516 }
517
518 multinomial_samples[sampling_type] = _multinomial(
@@ -481,25 +531,22 @@ def _sample_with_torch(
531
532 # GPU<->CPU sync happens in the loop below.
533 # This also converts the sample output to Python objects.
484 -
534 for sampling_type in SamplingType:
535 if sampling_type not in sample_metadata:
536 continue
488 - seq_group_ids, seq_groups, is_prompts, sample_indices = sample_metadata[
489 - sampling_type]
537 + (seq_group_id, seq_groups) = sample_metadata[sampling_type]
538 if sampling_type == SamplingType.GREEDY:
539 sample_results = _greedy_sample(seq_groups, greedy_samples)
540 elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
493 - sample_results = _random_sample(seq_groups, is_prompts,
541 + sample_results = _random_sample(seq_groups,
542 multinomial_samples[sampling_type])
543 elif sampling_type == SamplingType.BEAM:
496 - sample_results = _beam_search_sample(seq_groups, is_prompts,
497 - sampling_metadata.seq_data,
544 + sample_results = _beam_search_sample(seq_groups,
545 beam_search_logprobs)
499 - sample_results_dict.update(zip(seq_group_ids, sample_results))
546 + sample_results_dict.update(zip(seq_group_id, sample_results))
547
548 sample_results = [
502 - sample_results_dict[i]
549 + sample_results_dict.get(i, ([], []))
550 for i in range(len(sampling_metadata.seq_groups))
551 ]
552 return sample_results, sampled_token_ids_tensor
@@ -514,7 +561,7 @@ def _sample_with_triton_kernel(
561 categorized_seq_group_ids = {t: [] for t in SamplingType}
562 categorized_sample_indices = sampling_metadata.categorized_sample_indices
563 for i, seq_group in enumerate(sampling_metadata.seq_groups):
517 - _, sampling_params = seq_group
564 + sampling_params = seq_group.sampling_params
565 sampling_type = sampling_params.sampling_type
566 categorized_seq_group_ids[sampling_type].append(i)
567
@@ -530,17 +577,16 @@ def _sample_with_triton_kernel(
577 num_tokens = len(sample_indices)
578 if num_tokens == 0:
579 continue
533 - seq_group_ids = categorized_seq_group_ids[sampling_type]
534 - seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
535 - is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
536 - sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
537 - is_prompts, sample_indices,
580 + seq_group_id = categorized_seq_group_ids[sampling_type]
581 + seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_id]
582 + sample_metadata[sampling_type] = (seq_group_id, seq_groups,
583 + sample_indices,
584 sampled_token_indices)
585 if sampling_type in (SamplingType.GREEDY, SamplingType.RANDOM,
586 SamplingType.RANDOM_SEED):
541 - for seq_group, is_prompt in zip(seq_groups, is_prompts):
542 - if is_prompt:
543 - _, sampling_params = seq_group
587 + for seq_group in seq_groups:
588 + if seq_group.is_prompt:
589 + sampling_params = seq_group.sampling_params
590 max_best_of_in_batch = max(max_best_of_in_batch,
591 sampling_params.best_of)
592 elif sampling_type == SamplingType.BEAM:
@@ -564,22 +610,21 @@ def _sample_with_triton_kernel(
610 for sampling_type in SamplingType:
611 if sampling_type not in sample_metadata:
612 continue
567 - (seq_group_ids, seq_groups, is_prompts, sample_indices,
613 + (seq_group_id, seq_groups, sample_indices,
614 sampled_token_indices) = sample_metadata[sampling_type]
615 if sampling_type == SamplingType.GREEDY:
616 sample_results = _greedy_sample(
617 seq_groups, sampled_tokens[sampled_token_indices][:, 0])
618 elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
619 sample_results = _random_sample(
574 - seq_groups, is_prompts, sampled_tokens[sampled_token_indices])
620 + seq_groups, sampled_tokens[sampled_token_indices])
621 elif sampling_type == SamplingType.BEAM:
576 - sample_results = _beam_search_sample(seq_groups, is_prompts,
577 - sampling_metadata.seq_data,
622 + sample_results = _beam_search_sample(seq_groups,
623 beam_search_logprobs)
579 - sample_results_dict.update(zip(seq_group_ids, sample_results))
624 + sample_results_dict.update(zip(seq_group_id, sample_results))
625
626 sample_results = [
582 - sample_results_dict[i]
627 + sample_results_dict.get(i, ([], []))
628 for i in range(len(sampling_metadata.seq_groups))
629 ]
630 return sample_results
@@ -590,6 +635,18 @@ def _sample(
635 sampling_metadata: SamplingMetadata, sampling_tensors: SamplingTensors,
636 include_gpu_probs_tensor: bool, modify_greedy_probs: bool
637 ) -> Tuple[List[Tuple[List[int], List[int]]], Optional[torch.Tensor]]:
638 + """
639 + Args:
640 + probs: (num_query_tokens_in_batch, num_vocab)
641 + logprobs: (num_query_tokens_in_batch, num_vocab)
642 + sampling_metadata: The metadata for a batch for sampling.
643 + sampling_tensors: Tensors that include sampling related metadata.
644 +
645 + Returns:
646 + (next_token_ids, parent_seq_ids) for each seq group in a batch.
647 + If sampling is skipped, it returns ([], [])
648 + sampled_token_ids_tensor: A tensor of sampled token ids.
649 + """
650 return _sample_with_torch(
651 probs,
652 logprobs,
@@ -626,56 +683,97 @@ def _get_logprobs(
683 logprobs: torch.Tensor,
684 sampling_metadata: SamplingMetadata,
685 sample_results: List[Tuple[List[int], List[int]]],
629 -) -> Tuple[List[Optional[List[Optional[Dict[int, float]]]]], List[List[Dict[
630 - int, float]]]]:
631 - # Prepare query indices
632 - batched_logprobs_query_seq_indices: List[int] = []
633 - batched_logprobs_query_token_indices: List[int] = []
634 - # at least get one logprob for each token
686 +) -> Tuple[List[Optional[PromptLogprobs]], List[SampleLogprobs]]:
687 + """Return sample lobprobs and prompt logprobs.
688 +
689 + The logic consists of 3 parts.
690 + - Select indices to compute logprob from, ranks of token ids, and
691 + the top k token ids from logprobs.
692 + - Compute prompt logprobs if required.
693 + - Compute sample logprobs if required.
694 +
695 + Args:
696 + logprobs: (num_query_tokens_across_batch, num_vocab). Each query token's
697 + logprob per vocab. Sequence groups' query tokens are batched in a
698 + single flattened tensor. For example, assuming there are N
699 + seq groups, it is sorted by prefill tokens for seq_group_1 (if
700 + prompt logprob is enabled), decode tokens for seq_group_1 (if
701 + sampling is required), prefill tokens for seq_group_2, ...
702 + sampling_metadata: The sampling metadata.
703 + sample_results: (num_seq_groups) The tuple of (next_token_ids,
704 + parent_ids) for each sequence group. When beam search is enabled,
705 + sample_results can contain different number of seq_ids from
706 + sampling_metadata.seq_groups. It is because beam search creates
707 + 2 * BEAM_WIDTH number of samples (whereas there are only up to
708 + BEAM_WIDTH number of seq_ids).
709 +
710 + Returns:
711 + A tuple of prompt and sample logprobs per sequence group in a batch.
712 + """
713 + # The index of query token to calculate logprobs. It includes both
714 + # prompt and sample logprob indices.
715 + query_indices: List[int] = []
716 + # The next token ids to get the logprob value from.
717 + next_token_ids: List[int] = []
718 + # The largest requested number of logprobs. We find logprobs as many as the
719 + # largest num logprobs in this API.
720 largest_num_logprobs = 1
636 - sample_idx = 0
637 - for i, (seq_group, sample_result) in enumerate(
638 - zip(sampling_metadata.seq_groups, sample_results)):
639 - seq_ids, sampling_params = seq_group
640 - next_token_ids, parent_ids = sample_result
641 - num_parent_seqs = len(seq_ids)
642 - if (i < sampling_metadata.num_prompts
721 +
722 + # Select indices to compute logprob from, ranks of token ids, and the top
723 + # k token ids from logprobs.
724 + for (seq_group, sample_result) in zip(sampling_metadata.seq_groups,
725 + sample_results):
726 + sampling_params = seq_group.sampling_params
727 +
728 + # Update indices and tokens for prompt logprobs.
729 + if (seq_group.is_prompt
730 and sampling_params.prompt_logprobs is not None):
731 largest_num_logprobs = max(largest_num_logprobs,
732 sampling_params.prompt_logprobs)
646 - prompt_len = sampling_metadata.prompt_lens[i]
647 - prompt_tokens = sampling_metadata.seq_data[
648 - seq_ids[0]].prompt_token_ids
649 - batched_logprobs_query_seq_indices.extend(
650 - sample_idx + j for j in range(prompt_len - 1))
651 - batched_logprobs_query_token_indices.extend(
652 - token_id for token_id in prompt_tokens[1:])
653 - sample_idx += prompt_len - 1
654 - batched_logprobs_query_seq_indices.extend(
655 - [sample_idx + parent_id for parent_id in parent_ids])
656 - batched_logprobs_query_token_indices.extend(next_token_ids)
657 - if sampling_params.logprobs is not None:
658 - largest_num_logprobs = max(largest_num_logprobs,
659 - sampling_params.logprobs)
660 - sample_idx += num_parent_seqs
661 - assert sample_idx == logprobs.size(0)
662 -
663 - batched_logprobs_query_seq_indices_gpu = torch.tensor(
664 - batched_logprobs_query_seq_indices, device=logprobs.device)
665 - batched_logprobs_query_token_indices_gpu = torch.tensor(
666 - batched_logprobs_query_token_indices, device=logprobs.device)
667 -
668 - # Batched query for logprobs of selected token
669 - batched_logprobs_query_result = logprobs[[
670 - batched_logprobs_query_seq_indices_gpu,
671 - batched_logprobs_query_token_indices_gpu
733 + next_prompt_tokens = _get_next_prompt_tokens(seq_group)
734 + query_indices.extend(seq_group.prompt_logprob_indices)
735 + next_token_ids.extend(next_prompt_tokens)
736 +
737 + # Update indices and next tokenes for sample logprob.
738 + if seq_group.do_sample:
739 + token_ids, parent_seq_ids = sample_result
740 + # NOTE: We cannot directly use sample_indices because
741 + # sample_indices only contain parent seq_ids of a previous step.
742 + # The current step may have different number of seq_ids, and
743 + # we can obtain it from `sample_result[1]`.
744 + query_idx = seq_group.sample_indices[0]
745 + query_indices.extend(
746 + [query_idx + parent_id for parent_id in parent_seq_ids])
747 + next_token_ids.extend(token_ids)
748 +
749 + if sampling_params.logprobs is not None:
750 + largest_num_logprobs = max(largest_num_logprobs,
751 + sampling_params.logprobs)
752 +
753 + assert len(next_token_ids) == len(query_indices)
754 +
755 + if len(query_indices) == 0:
756 + empty_sampled_logprob = []
757 + empty_prompt_logprob = None
758 + return [empty_prompt_logprob], [empty_sampled_logprob]
759 +
760 + query_indices_gpu = torch.tensor(query_indices, device=logprobs.device)
761 + next_token_ids_gpu = torch.tensor(next_token_ids, device=logprobs.device)
762 +
763 + # (num_selected_query_tokens, num_logprobs). Note that query_indices can
764 + # contain duplicates if beam search is enabled.
765 + selected_logprobs = logprobs[[
766 + query_indices_gpu,
767 + next_token_ids_gpu,
768 ]]
769 + ranks = _get_ranks(
770 + logprobs[query_indices_gpu],
771 + next_token_ids_gpu,
772 + )
773 + assert selected_logprobs.shape[0] == ranks.shape[0]
774
674 - batched_ranks_query_result = _get_ranks(
675 - logprobs[batched_logprobs_query_seq_indices_gpu],
676 - batched_logprobs_query_token_indices_gpu)
677 -
678 - # Batched query for logprobs of topk tokens
775 + # Logprobs of topk tokens for a batch of sequence groups.
776 + # (num_query_tokens_across_batch).
777 if largest_num_logprobs > 0:
778 top_logprobs, top_token_ids = torch.topk(logprobs,
779 largest_num_logprobs,
@@ -685,79 +783,136 @@ def _get_logprobs(
783 else:
784 top_logprobs, top_token_ids = None, None
785
688 - batched_logprobs_query_result = batched_logprobs_query_result.cpu()
689 - batched_ranks_query_result = batched_ranks_query_result.cpu()
690 -
691 - # Gather results
692 - result_prompt_logprobs: List[Optional[PromptLogprobs]] = []
693 - result_sample_logprobs: List[SampleLogprobs] = []
694 - sample_idx = 0
695 - query_result_idx = 0
696 - for i, (seq_group, sample_result) in enumerate(
697 - zip(sampling_metadata.seq_groups, sample_results)):
698 - seq_ids, sampling_params = seq_group
699 - next_token_ids, parent_ids = sample_result
786 + selected_logprobs = selected_logprobs.cpu()
787 + ranks = ranks.cpu()
788 +
789 + # Find prompt/sample logprobs.
790 + prompt_logprobs_per_seq_group: List[Optional[PromptLogprobs]] = []
791 + sample_logprobs_per_seq_group: List[SampleLogprobs] = []
792 + top_logprob_idx = 0
793 + selected_logprobs_idx = 0
794 +
795 + for seq_group, sample_result in zip(sampling_metadata.seq_groups,
796 + sample_results):
797 + (prompt_logprobs, top_logprob_idx,
798 + selected_logprobs_idx) = _get_prompt_logprob_if_needed(
799 + seq_group, selected_logprobs, ranks, top_token_ids, top_logprobs,
800 + selected_logprobs_idx, top_logprob_idx)
801 + prompt_logprobs_per_seq_group.append(prompt_logprobs)
802 +
803 + (sampled_logprobs, top_logprob_idx,
804 + selected_logprobs_idx) = _get_sampled_logprob_if_needed(
805 + seq_group, sample_result, selected_logprobs, ranks, top_token_ids,
806 + top_logprobs, selected_logprobs_idx, top_logprob_idx)
807 + sample_logprobs_per_seq_group.append(sampled_logprobs)
808 +
809 + return prompt_logprobs_per_seq_group, sample_logprobs_per_seq_group
810 +
811 +
812 +def _get_prompt_logprob_if_needed(
813 + seq_group: SequenceGroupToSample,
814 + selected_logprobs: torch.Tensor,
815 + ranks: torch.Tensor,
816 + top_token_ids: torch.Tensor,
817 + top_logprobs: torch.Tensor,
818 + selected_logprobs_idx: int,
819 + top_logprob_idx: int,
820 +):
821 + """Compute the prompt logprob from a sequence group if needed."""
822 + sampling_params = seq_group.sampling_params
823 + is_prompt = seq_group.is_prompt
824 +
825 + # Find prompt logprobs
826 + prompt_logprobs: Optional[PromptLogprobs] = None
827 + if (is_prompt and sampling_params.prompt_logprobs is not None):
828 + prompt_logprobs = []
829 + num_logprobs = sampling_params.prompt_logprobs
830 + next_prompt_tokens = _get_next_prompt_tokens(seq_group)
831 + for token_id in next_prompt_tokens:
832 + # Calculate the prompt logprob of the real prompt tokens.
833 + # Use tuple here for performance (to use to_list()).
834 + # {token_id: (logprob, rank_from_vocab)}
835 + prompt_logprobs_dict: Dict[int, Tuple[float, int]] = {
836 + token_id: (selected_logprobs[selected_logprobs_idx].item(),
837 + ranks[selected_logprobs_idx].item())
838 + }
839
701 - # Prompt logprobs
702 - if (i < sampling_metadata.num_prompts
703 - and sampling_params.prompt_logprobs is not None):
704 - num_logprobs = sampling_params.prompt_logprobs
705 - prompt_tokens = sampling_metadata.seq_data[
706 - seq_ids[0]].prompt_token_ids
707 - group_prompt_logprobs: PromptLogprobs = [None]
708 - for token_id in prompt_tokens[1:]:
709 - prompt_logprobs_dict = {
710 - token_id:
711 - (batched_logprobs_query_result[query_result_idx].item(),
712 - batched_ranks_query_result[query_result_idx].item())
713 - }
714 - if num_logprobs > 0:
715 - prompt_logprobs_dict.update(
840 + # Add top K prompt logprobs along with its rank.
841 + if num_logprobs > 0:
842 + prompt_logprobs_dict.update(
843 + zip(
844 + top_token_ids[top_logprob_idx, :num_logprobs].tolist(),
845 zip(
717 - top_token_ids[sample_idx, :num_logprobs].tolist(),
718 - zip(
719 - top_logprobs[
720 - sample_idx, :num_logprobs].tolist(),
721 - range(1, num_logprobs + 1))))
722 - group_prompt_logprobs.append({
723 - token_id: Logprob(*logprob_rank)
724 - for token_id, logprob_rank in prompt_logprobs_dict.items()
725 - })
726 - sample_idx += 1
727 - query_result_idx += 1
728 - result_prompt_logprobs.append(group_prompt_logprobs)
729 - else:
730 - result_prompt_logprobs.append(None)
731 -
732 - # Sample logprobs
733 - num_logprobs = sampling_params.logprobs
734 - if num_logprobs is None:
735 - num_logprobs = 0
736 - group_sample_logprobs: SampleLogprobs = []
737 - for next_token_id, parent_id in zip(next_token_ids, parent_ids):
738 - sample_logprobs_dict = {
846 + top_logprobs[
847 + top_logprob_idx, :num_logprobs].tolist(),
848 + # This is ranks. Since top_logprob is sorted,
849 + # we can just use a range here.
850 + range(1, num_logprobs + 1))))
851 + prompt_logprobs.append({
852 + token_id: Logprob(*logprob_and_rank)
853 + for token_id, logprob_and_rank in prompt_logprobs_dict.items()
854 + })
855 + # + 1 to go to the next prompt token.
856 + top_logprob_idx += 1
857 + selected_logprobs_idx += 1
858 + return prompt_logprobs, top_logprob_idx, selected_logprobs_idx
859 +
860 +
861 +def _get_sampled_logprob_if_needed(
862 + seq_group: SequenceGroupToSample,
863 + sample_result: Tuple[List[int], List[int]],
864 + selected_logprobs: torch.Tensor,
865 + ranks: torch.Tensor,
866 + top_token_ids: torch.Tensor,
867 + top_logprobs: torch.Tensor,
868 + selected_logprobs_idx: int,
869 + top_logprob_idx: int,
870 +):
871 + """Compute the sample logprob if needed."""
872 + seq_ids = seq_group.seq_ids
873 + num_logprobs = seq_group.sampling_params.logprobs
874 + if num_logprobs is None:
875 + num_logprobs = 0
876 + sampled_logprobs: SampleLogprobs = []
877 + next_token_ids, parent_seq_ids = sample_result
878 +
879 + if seq_group.do_sample:
880 + assert len(next_token_ids) > 0
881 + for (next_token_id, parent_id) in zip(next_token_ids, parent_seq_ids):
882 + # Calculate the sample logprob of the real sampled tokens.
883 + # Use tuple here for performance (to use to_list()).
884 + # token_id: (logprob, rank_from_vocab)
885 + sampled_logprobs_dict: Dict[int, Tuple[float, int]] = {
886 next_token_id:
740 - (batched_logprobs_query_result[query_result_idx].item(),
741 - batched_ranks_query_result[query_result_idx].item())
887 + (selected_logprobs[selected_logprobs_idx].item(),
888 + ranks[selected_logprobs_idx].item())
889 }
743 - query_result_idx += 1
890 + # +1 to go to the next sampled token. Note that
891 + # selected_logprobs can contain duplicates unlike top_logprobs
892 + # when beam search is enabled.
893 + selected_logprobs_idx += 1
894 +
895 + # Second, add top K logprobs along with its rank.
896 if num_logprobs >= 0:
745 - sample_logprobs_dict.update(
897 + sampled_logprobs_dict.update(
898 zip(
747 - top_token_ids[sample_idx +
899 + top_token_ids[top_logprob_idx +
900 parent_id, :num_logprobs].tolist(),
901 zip(
750 - top_logprobs[sample_idx +
902 + top_logprobs[top_logprob_idx +
903 parent_id, :num_logprobs].tolist(),
904 + # This is rank. Since top_logprob is sorted, we
905 + # can just use a range here.
906 range(1, num_logprobs + 1))))
753 - group_sample_logprobs.append({
754 - token_id: Logprob(*logprob_rank)
755 - for token_id, logprob_rank in sample_logprobs_dict.items()
907 + sampled_logprobs.append({
908 + token_id: Logprob(*logprob_and_rank)
909 + for token_id, logprob_and_rank in
910 + sampled_logprobs_dict.items()
911 })
757 - result_sample_logprobs.append(group_sample_logprobs)
758 - sample_idx += len(seq_ids)
759 -
760 - return result_prompt_logprobs, result_sample_logprobs
912 + # There are len(seq_ids) number of sampled tokens for the current
913 + # sequence group in top_logprobs. Jump to the next seq_group.
914 + top_logprob_idx += len(seq_ids)
915 + return sampled_logprobs, top_logprob_idx, selected_logprobs_idx
916
917
918 def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
@@ -832,7 +987,7 @@ def _build_sampler_output(
987 group_sample_logprobs) in zip(sampling_metadata.seq_groups,
988 sample_results, prompt_logprobs,
989 sample_logprobs):
835 - seq_ids, _ = seq_group
990 + seq_ids = seq_group.seq_ids
991 next_token_ids, parent_ids = sample_result
992 seq_outputs = []
993 for parent_id, next_token_id, logprobs in zip(parent_ids,
@@ -854,3 +1009,36 @@ def _build_sampler_output(
1009 sampled_token_probs=sampled_token_probs,
1010 sampled_token_ids=sampled_token_ids,
1011 )
1012 +
1013 +
1014 +def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[str]:
1015 + """Get a list of next prompt tokens to compute logprob from a
1016 + given sequence group.
1017 +
1018 + It is used to compute prompt logprob. Imagine you have logprob for each
1019 + query token. Query token needs to know the next prompt token id to compute
1020 + prompt logprob. This is a helper to obtain next prompt token ids.
1021 +
1022 + This API has to be used only when the caller knows seq_group is in prefill
1023 + stage.
1024 +
1025 + Returns:
1026 + A list of next prompt tokens to compute logprob.
1027 + """
1028 + assert seq_group.is_prompt, (
1029 + "Caller should ensure the sequence group is in a prefill stage.")
1030 + seq_ids = seq_group.seq_ids
1031 + subquery_len = seq_group.subquery_len
1032 + assert subquery_len is not None
1033 + # prompt has only 1 seq id.
1034 + assert len(seq_ids) == 1
1035 + seq_data = seq_group.seq_data[seq_ids[0]]
1036 + computed_len = seq_data.get_num_computed_tokens()
1037 + prompt_tokens = seq_data.prompt_token_ids
1038 + # +1 because we are looking for a next prompt token.
1039 + next_token_index_start = computed_len + 1
1040 + next_token_index_end = min(computed_len + subquery_len + 1,
1041 + len(prompt_tokens))
1042 + next_prompt_tokens = prompt_tokens[
1043 + next_token_index_start:next_token_index_end]
1044 + return next_prompt_tokens
vllm/model_executor/sampling_metadata.py
+284 -65
@@ -6,57 +6,275 @@ import torch
6
7 from vllm.model_executor.layers.ops.sample import get_num_triton_sampler_splits
8 from vllm.sampling_params import SamplingParams, SamplingType
9 -from vllm.sequence import SequenceData
10 -from vllm.utils import is_pin_memory_available
9 +from vllm.sequence import SequenceData, SequenceGroupMetadata
10 +from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
11 + maybe_expand_dim)
12
13 _SAMPLING_EPS = 1e-5
14 _SEED_0_REPLACEMENT = 3403598558
15
16
17 +@dataclass
18 +class SequenceGroupToSample:
19 + # Sequence ids for the sequence group in a previous step.
20 + seq_ids: List[int]
21 + sampling_params: SamplingParams
22 + # seq_id -> sequence data.
23 + seq_data: Dict[int, SequenceData]
24 + # The length of the prompt of the sequence group. None if it is in a decode
25 + # stage.
26 + prompt_len: Optional[int]
27 + # The length of the query tokens to compute in the current step. None if it
28 + # is in a decode stage. The length of subquery_len <= prompt_len.
29 + subquery_len: Optional[int]
30 + # A random number generator for sampling.
31 + generator: Optional[torch.Generator]
32 + # True if the sequence group is in prefill stage. False if it is in a
33 + # decode stage.
34 + is_prompt: bool
35 + # Query token indices from logits. to compute prompt logprob. Empty if
36 + # prompt logprob is not required.
37 + prompt_logprob_indices: List[int]
38 + # Sample token indices from logits. Empty if sampling is not required.
39 + sample_indices: List[int]
40 +
41 + @property
42 + def do_sample(self):
43 + return len(self.sample_indices) > 0
44 +
45 + def __post_init__(self):
46 + if len(self.prompt_logprob_indices) > 0:
47 + assert self.sampling_params.prompt_logprobs is not None
48 + if self.is_prompt:
49 + assert self.prompt_len is not None
50 + assert self.subquery_len is not None
51 +
52 +
53 class SamplingMetadata:
54 """Metadata for input sequences. Used in sampler.
55
56 + The usage is as follow;
57 + ```
58 + hidden_states = execute_model(...)
59 + logits = hidden_states[sampling_metadata.selected_token_indices]
60 + sample(logits)
61 +
62 + def sample(logits):
63 + # Use categorized_sample_indices for sampling....
64 + ```
65 +
66 Args:
20 - seq_groups: List of (seq_ids, sampling_params).
21 - seq_data: Seq_id -> SequenceData.
22 - prompt_lens: Lengths of prompts.
23 - selected_token_indices: Token indices selected for sampling.
67 + seq_groups: List of batched sequence groups.
68 + selected_token_indices: (num_query_tokens_to_logprob). Indices to find
69 + logits from the initial model output hidden states.
70 categorized_sample_indices: SamplingType -> token indices to sample.
25 - generators: List of torch.Generators to use for seeded sampling
26 - perform_sampling: Whether to perform sampling. This option is used to
27 - make the sampling only happens in the driver worker, and disable
28 - sampling in other worker processes.
71 + Each token indices is 2D tensor of (num_indices, num_indices) where
72 + the first item means the sample index within the returned logit
73 + (before pruning padding), and the second item means the sample
74 + index after pruning using selected_token_indices.
75 + For example, if the returned logit is [1, 2, 3], and we select
76 + [1, 2] for sampling, the pruned logit will be [2, 3]. In this case,
77 + The first tuple is [1, 2] (sampled index within original logit),
78 + and the second tuple is [0, 1] (sampled index within pruned logit).
79 + num_prompts: Number of prompt sequence groups in seq_groups.
80 """
81
82 def __init__(
83 self,
33 - seq_groups: Optional[List[Tuple[List[int], SamplingParams]]],
34 - seq_data: Optional[Dict[int, SequenceData]],
35 - prompt_lens: Optional[List[int]],
84 + seq_groups: List[SequenceGroupToSample],
85 selected_token_indices: torch.Tensor,
37 - categorized_sample_indices: Optional[Dict[SamplingType, torch.Tensor]],
38 - generators: Optional[List[torch.Generator]] = None,
39 - perform_sampling: bool = True,
86 + categorized_sample_indices: Dict[SamplingType, torch.Tensor],
87 + num_prompts: int,
88 ) -> None:
89 self.seq_groups = seq_groups
42 - self.seq_data = seq_data
43 - self.prompt_lens = prompt_lens
90 self.selected_token_indices = selected_token_indices
91 self.categorized_sample_indices = categorized_sample_indices
46 - self.generators = generators
47 - self.perform_sampling = perform_sampling
92 + self.num_prompts = num_prompts
93
49 - self.num_prompts = len(prompt_lens) if prompt_lens is not None else 0
94 + @staticmethod
95 + def prepare(
96 + seq_group_metadata_list: List[SequenceGroupMetadata],
97 + prompt_lens: List[int],
98 + subquery_lens: Optional[List[int]],
99 + device: str,
100 + pin_memory: bool,
101 + ) -> "SamplingMetadata":
102 + (
103 + seq_groups,
104 + selected_token_indices,
105 + categorized_sample_indices,
106 + num_prompts,
107 + ) = _prepare_seq_groups(seq_group_metadata_list, prompt_lens,
108 + subquery_lens, device)
109 + selected_token_indices = async_tensor_h2d(selected_token_indices,
110 + dtype=torch.long,
111 + target_device=device,
112 + pin_memory=pin_memory)
113 + categorized_sample_indices = {
114 + t: maybe_expand_dim(
115 + async_tensor_h2d(seq_ids,
116 + dtype=torch.int,
117 + target_device=device,
118 + pin_memory=pin_memory), 2, 2)
119 + for t, seq_ids in categorized_sample_indices.items()
120 + }
121 +
122 + sampling_metadata = SamplingMetadata(
123 + seq_groups=seq_groups,
124 + selected_token_indices=selected_token_indices,
125 + categorized_sample_indices=categorized_sample_indices,
126 + num_prompts=num_prompts,
127 + )
128 + return sampling_metadata
129
130 def __repr__(self) -> str:
131 return (
132 "SamplingMetadata("
133 f"seq_groups={self.seq_groups}, "
55 - f"seq_data={self.seq_data}, "
56 - f"prompt_lens={self.prompt_lens}, "
134 f"selected_token_indices={self.selected_token_indices}, "
58 - f"categorized_sample_indices={self.categorized_sample_indices}), "
59 - f"perform_sampling={self.perform_sampling})")
135 + f"categorized_sample_indices={self.categorized_sample_indices}), ")
136 +
137 +
138 +def _prepare_seq_groups(
139 + seq_group_metadata_list: List[SequenceGroupMetadata],
140 + prompt_lens: List[int],
141 + subquery_lens: Optional[List[int]],
142 + device: str,
143 +) -> Tuple[List[SequenceGroupToSample], List[int], Dict[
144 + SamplingType, List[Tuple[int, int]]], int]:
145 + """Prepare sequence groups and indices for sampling.
146 +
147 + Args:
148 + seq_group_metadata_list: A list of sequence group to batch.
149 + prompt_lens: A list of prompt lens per sequence group.
150 + Index of prompt len should match with seq_group_metadata_list.
151 + subquery_lens: A list of query lengths. Prompt lens include the length
152 + of entire prompt tokens, and it could be shorter.
153 + device: A device to use for random number generator,
154 + `SequenceGroupToSample.generator`.
155 +
156 + Returns:
157 + seq_groups: A list of sequence group to sample.
158 + selected_token_indices: See the definition from `SamplingMetadata`.
159 + categorized_sample_indices: See the definition from `SamplingMetadata`.
160 + num_prompts: Total number of prompts from `seq_group_metadata_list`.
161 + """
162 + # Batched sequence groups for the current model forward stsep.
163 + seq_groups: List[SequenceGroupToSample] = []
164 + # A list of token indices to sample/compute logprob. It is used to
165 + # prune the outcome logits from the model for the performance.
166 + selected_token_indices: List[int] = []
167 + # Used for selected_token_indices.
168 + model_output_idx = 0
169 +
170 + # Sampling type -> (
171 + # indices to sample/prompt logprob within pruned output logits,
172 + # indices to sample within pruned logits)
173 + categorized_sample_indices: Dict[SamplingType, List[Tuple[int, int]]] = {
174 + t: []
175 + for t in SamplingType
176 + }
177 + # Index of logits to compute logprob. Logits include both prompt logprob
178 + # and sample logprob indices.
179 + logit_idx = 0
180 + # Index to sample from a sample tensor. It is used by triton sample kernel.
181 + # See `_sample_with_triton_kernel` for more details.
182 + sample_idx = 0
183 + # Total number of prompts from given sequence groups.
184 + num_prompts = 0
185 +
186 + for i, seq_group_metadata in enumerate(seq_group_metadata_list):
187 + seq_ids = list(seq_group_metadata.seq_data.keys())
188 + sampling_params = seq_group_metadata.sampling_params
189 + is_prompt = seq_group_metadata.is_prompt
190 + generator: Optional[torch.Generator] = None
191 + # If the current seq group is in decode stage, it is None.
192 + prompt_len: Optional[int] = None
193 + subquery_len: Optional[int] = None
194 + prompt_logprob_indices: List[int] = []
195 + sample_indices: List[int] = []
196 + do_sample = seq_group_metadata.do_sample
197 +
198 + if seq_group_metadata.is_prompt:
199 + if sampling_params.seed is not None:
200 + seq_group_metadata.state.generator = torch.Generator(
201 + device=device).manual_seed(sampling_params.seed)
202 +
203 + num_prompts += 1
204 + num_prefill_sample = len(seq_ids)
205 + assert num_prefill_sample == 1
206 + assert subquery_lens is not None and prompt_lens is not None
207 + subquery_len, prompt_len = subquery_lens[i], prompt_lens[i]
208 + # If we need sampling, exclude num_prefill_sample tokens from
209 + # prompt logprob.
210 + prompt_logprob_len = (subquery_len - num_prefill_sample
211 + if do_sample else subquery_len)
212 + sample_len = num_prefill_sample if do_sample else 0
213 + else:
214 + # Decode
215 + prompt_logprob_len = 0
216 + sample_len = len(seq_ids) if do_sample else 0
217 +
218 + # Update indices to select from the model output.
219 + """
220 + This blocks computes selected_token_indices which is used in the
221 + following way.
222 +
223 + hidden_states = model(...)
224 + logits = hidden_states[selected_token_indices]
225 + """
226 +
227 + if sampling_params.prompt_logprobs:
228 + selected_token_indices.extend(
229 + range(model_output_idx, model_output_idx + prompt_logprob_len))
230 + model_output_idx += prompt_logprob_len
231 + if do_sample:
232 + selected_token_indices.extend(
233 + range(model_output_idx, model_output_idx + sample_len))
234 + model_output_idx += sample_len
235 +
236 + # We now find indices for logprob computation and sampling.
237 + """
238 + This block computes categorized_sample_indices which is used in the
239 + following way.
240 +
241 + hidden_states = model(...)
242 + logits = hidden_states[selected_token_indices]
243 + def sample(logits):
244 + # Use categorized_sample_indices for sampling.
245 + # prompt_logprob_indices to find prompt logprob indices.
246 + # sample_indices to find sample indices.
247 + """
248 +
249 + if sampling_params.prompt_logprobs is not None:
250 + prompt_logprob_indices.extend(
251 + range(logit_idx, logit_idx + prompt_logprob_len))
252 + logit_idx += prompt_logprob_len
253 + if do_sample:
254 + sample_indices.extend(range(logit_idx, logit_idx + sample_len))
255 + categorized_sample_indices[sampling_params.sampling_type].extend(
256 + list(
257 + zip(range(logit_idx, logit_idx + sample_len),
258 + range(sample_idx, sample_idx + sample_len))))
259 + logit_idx += sample_len
260 + sample_idx += sample_len
261 +
262 + if sampling_params.seed is not None:
263 + generator = seq_group_metadata.state.generator
264 +
265 + seq_groups.append(
266 + SequenceGroupToSample(
267 + seq_ids=seq_ids,
268 + sampling_params=sampling_params,
269 + seq_data=seq_group_metadata.seq_data,
270 + prompt_len=prompt_len,
271 + subquery_len=subquery_len,
272 + generator=generator,
273 + is_prompt=is_prompt,
274 + prompt_logprob_indices=list(prompt_logprob_indices),
275 + sample_indices=list(sample_indices)))
276 + return (seq_groups, selected_token_indices, categorized_sample_indices,
277 + num_prompts)
278
279
280 @dataclass
@@ -112,11 +330,10 @@ class SamplingTensors:
330 seeds_to_generate = (extra_seeds_to_generate +
331 get_num_triton_sampler_splits(vocab_size))
332
115 - sample_indices_start_idx = 0
333 assert sampling_metadata.seq_groups is not None
117 - assert sampling_metadata.seq_data is not None
118 - for i, seq_group in enumerate(sampling_metadata.seq_groups):
119 - seq_ids, sampling_params = seq_group
334 + for seq_group in sampling_metadata.seq_groups:
335 + seq_ids = seq_group.seq_ids
336 + sampling_params = seq_group.sampling_params
337 temperature = sampling_params.temperature
338 p = sampling_params.presence_penalty
339 f = sampling_params.frequency_penalty
@@ -145,45 +362,46 @@ class SamplingTensors:
362 or abs(r - 1.0) >= _SAMPLING_EPS):
363 do_penalties = True
364
148 - if (i < sampling_metadata.num_prompts
365 + is_prompt = seq_group.is_prompt
366 + if (seq_group.is_prompt
367 and sampling_params.prompt_logprobs is not None):
368 # For tokens in the prompt that we only need to get
369 # their logprobs
152 - assert sampling_metadata.prompt_lens is not None
153 - prompt_len = sampling_metadata.prompt_lens[i]
154 - temperatures += [temperature] * (prompt_len - 1)
155 - top_ps += [top_p] * (prompt_len - 1)
156 - top_ks += [top_k] * (prompt_len - 1)
157 - min_ps += [min_p] * (prompt_len - 1)
158 - presence_penalties += [0] * (prompt_len - 1)
159 - frequency_penalties += [0] * (prompt_len - 1)
160 - repetition_penalties += [1] * (prompt_len - 1)
161 - prompt_tokens.extend([] for _ in range(prompt_len - 1))
162 - output_tokens.extend([] for _ in range(prompt_len - 1))
163 - for seq_id in seq_ids:
164 - seq_data = sampling_metadata.seq_data[seq_id]
165 - prompt_tokens.append(seq_data.prompt_token_ids)
166 - output_tokens.append(seq_data.output_token_ids)
167 - temperatures += [temperature] * len(seq_ids)
168 - top_ps += [top_p] * len(seq_ids)
169 - top_ks += [top_k] * len(seq_ids)
170 - min_ps += [min_p] * len(seq_ids)
171 - presence_penalties += [p] * len(seq_ids)
172 - frequency_penalties += [f] * len(seq_ids)
173 - repetition_penalties += [r] * len(seq_ids)
174 -
175 - is_prompt = i < sampling_metadata.num_prompts
370 + subquery_len = seq_group.subquery_len
371 + assert subquery_len is not None
372 + prefill_len = len(seq_group.prompt_logprob_indices)
373 + temperatures += [temperature] * prefill_len
374 + top_ps += [top_p] * prefill_len
375 + top_ks += [top_k] * prefill_len
376 + min_ps += [min_p] * prefill_len
377 + presence_penalties += [0] * prefill_len
378 + frequency_penalties += [0] * prefill_len
379 + repetition_penalties += [1] * prefill_len
380 + prompt_tokens.extend([] for _ in range(prefill_len))
381 + output_tokens.extend([] for _ in range(prefill_len))
382 +
383 + if seq_group.do_sample:
384 + sample_lens = len(seq_group.sample_indices)
385 + assert sample_lens == len(seq_ids)
386 + for seq_id in seq_ids:
387 + seq_data = seq_group.seq_data[seq_id]
388 + prompt_tokens.append(seq_data.prompt_token_ids)
389 + output_tokens.append(seq_data.output_token_ids)
390 + temperatures += [temperature] * len(seq_ids)
391 + top_ps += [top_p] * len(seq_ids)
392 + top_ks += [top_k] * len(seq_ids)
393 + min_ps += [min_p] * len(seq_ids)
394 + presence_penalties += [p] * len(seq_ids)
395 + frequency_penalties += [f] * len(seq_ids)
396 + repetition_penalties += [r] * len(seq_ids)
397 +
398 if is_prompt:
399 prompt_best_of.append(sampling_params.best_of)
178 - assert sampling_metadata.prompt_lens is not None
179 - prompt_len = sampling_metadata.prompt_lens[i]
400 + subquery_len = seq_group.subquery_len
401 + assert subquery_len is not None
402
181 - if sampling_params.prompt_logprobs is not None:
182 - # NOTE: the sampling position is the last token
183 - # in the prompt
184 - sample_indices_start_idx += prompt_len - 1
403 for seq_id in seq_ids:
186 - seq_data = sampling_metadata.seq_data[seq_id]
404 + seq_data = seq_group.seq_data[seq_id]
405 extra_entropy = extra_entropy or ()
406 seq_seeds = cls._get_sequence_seeds(
407 seed,
@@ -193,8 +411,7 @@ class SamplingTensors:
411 seeds_to_generate=seeds_to_generate,
412 is_greedy=is_greedy)
413 sampling_seeds.append(seq_seeds)
196 - sample_indices.append(sample_indices_start_idx)
197 - sample_indices_start_idx += 1
414 + sample_indices.extend(seq_group.sample_indices)
415
416 sampling_tensors = SamplingTensors.from_lists(
417 temperatures, top_ps, top_ks, min_ps, presence_penalties,
@@ -217,12 +434,14 @@ class SamplingTensors:
434 # Note that the performance will be very bad without
435 # pinned memory.
436 pin_memory = is_pin_memory_available()
220 - prompt_max_len = max(len(tokens) for tokens in prompt_tokens)
437 + prompt_max_len = max([len(tokens) for tokens in prompt_tokens],
438 + default=0)
439 prompt_padded_tokens = [
440 tokens + [vocab_size] * (prompt_max_len - len(tokens))
441 for tokens in prompt_tokens
442 ]
225 - output_max_len = max(len(tokens) for tokens in output_tokens)
443 + output_max_len = max([len(tokens) for tokens in output_tokens],
444 + default=0)
445 output_padded_tokens = [
446 tokens + [vocab_size] * (output_max_len - len(tokens))
447 for tokens in output_tokens
vllm/sequence.py
+10 -1
@@ -28,7 +28,10 @@ class Logprob:
28 decoded_token: Optional[str] = None
29
30
31 +# {token_id -> logprob} per each sequence group. None if the corresponding
32 +# sequence group doesn't require prompt logprob.
33 PromptLogprobs = List[Optional[Dict[int, Logprob]]]
34 +# {token_id -> logprob} for each sequence group.
35 SampleLogprobs = List[Dict[int, Logprob]]
36
37
@@ -215,7 +218,7 @@ class Sequence:
218 self.eos_token_id = eos_token_id
219 self.lora_request = lora_request
220
218 - self.data = SequenceData(prompt_token_ids)
221 + self.data: SequenceData = SequenceData(prompt_token_ids)
222 self.output_logprobs: SampleLogprobs = []
223 self.output_text = ""
224
@@ -559,6 +562,9 @@ class SequenceGroupMetadata:
562 sampling_params: The sampling parameters used to generate the outputs.
563 block_tables: The block tables. (Seq id -> list of physical block
564 numbers)
565 + do_sample: True if sampling is required. Sampling is not required when
566 + e.g., prefill is chunked, and the current iteration only computes
567 + query tokens for prefill, we don't need sampling.
568 token_chunk_size: The number of tokens to be processed (per sequence).
569 None if chunking is not required.
570 state: Internal state tied to this sequence group.
@@ -573,6 +579,7 @@ class SequenceGroupMetadata:
579 seq_data: Dict[int, SequenceData],
580 sampling_params: SamplingParams,
581 block_tables: Dict[int, List[int]],
582 + do_sample: bool = True,
583 token_chunk_size: Optional[int] = None,
584 lora_request: Optional[LoRARequest] = None,
585 computed_block_nums: Optional[List[int]] = None,
@@ -589,6 +596,7 @@ class SequenceGroupMetadata:
596 self.multi_modal_data = multi_modal_data
597 self.state = SequenceGroupState() if state is None else state
598 self._token_chunk_size = token_chunk_size
599 + self.do_sample = do_sample
600
601 if self._token_chunk_size is None:
602 if is_prompt:
@@ -650,6 +658,7 @@ class SequenceGroupOutput:
658 prompt_logprobs: Optional[PromptLogprobs],
659 ) -> None:
660 self.samples = samples
661 + # Prompt logprob for each prompt query token.
662 self.prompt_logprobs = prompt_logprobs
663
664 def __repr__(self) -> str:
vllm/worker/cpu_model_runner.py
+15 -101
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Optional, Tuple
1 +from typing import List, Optional, Tuple
2
3 import torch
4 from torch import nn
@@ -10,9 +10,8 @@ from vllm.distributed import broadcast_tensor_dict
10 from vllm.logger import init_logger
11 from vllm.model_executor import SamplingMetadata
12 from vllm.model_executor.model_loader import get_model
13 -from vllm.sampling_params import SamplingParams, SamplingType
14 -from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15 -from vllm.utils import make_tensor_with_pad, maybe_expand_dim
13 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
14 +from vllm.utils import make_tensor_with_pad
15
16 logger = init_logger(__name__)
17
@@ -38,6 +37,8 @@ class CPUModelRunner:
37 self.model_config = model_config
38 self.parallel_config = parallel_config
39 self.scheduler_config = scheduler_config
40 + # Currently, CPU worker doesn't support chunked prefill.
41 + assert self.scheduler_config.chunked_prefill_enabled is False
42 self.lora_config = lora_config
43 self.vision_language_config = vision_language_config
44 self.load_config = load_config
@@ -252,99 +253,6 @@ class CPUModelRunner:
253 attn_metadata,
254 )
255
255 - def _prepare_sample(
256 - self,
257 - seq_group_metadata_list: List[SequenceGroupMetadata],
258 - prompt_lens: List[int],
259 - ) -> SamplingMetadata:
260 - seq_groups: List[Tuple[List[int], SamplingParams]] = []
261 - selected_token_indices: List[int] = []
262 - generators: List[torch.Generator] = []
263 - selected_token_start_idx = 0
264 - categorized_sample_indices: Dict[SamplingType,
265 - List[Tuple[int, int]]] = {
266 - t: []
267 - for t in SamplingType
268 - }
269 - categorized_sample_indices_start_idx = 0
270 - categorized_sampled_token_indices_start_idx = 0
271 -
272 - for i, seq_group_metadata in enumerate(seq_group_metadata_list):
273 - seq_ids = list(seq_group_metadata.seq_data.keys())
274 - sampling_params = seq_group_metadata.sampling_params
275 - seq_groups.append((seq_ids, sampling_params))
276 -
277 - if seq_group_metadata.is_prompt:
278 - assert len(seq_ids) == 1
279 - subquery_len = prompt_lens[i]
280 - if sampling_params.prompt_logprobs is not None:
281 - # NOTE: prompt token positions do not need sample, skip
282 - categorized_sample_indices_start_idx += subquery_len - 1
283 -
284 - categorized_sample_indices[
285 - sampling_params.sampling_type].append(
286 - (categorized_sample_indices_start_idx,
287 - categorized_sampled_token_indices_start_idx))
288 - categorized_sample_indices_start_idx += 1
289 - categorized_sampled_token_indices_start_idx += 1
290 -
291 - if sampling_params.prompt_logprobs is not None:
292 - selected_token_indices.extend(
293 - range(selected_token_start_idx,
294 - selected_token_start_idx + subquery_len - 1))
295 - selected_token_indices.append(selected_token_start_idx +
296 - subquery_len - 1)
297 - selected_token_start_idx += subquery_len
298 -
299 - if sampling_params.seed is not None:
300 - seq_group_metadata.state.generator = torch.Generator(
301 - device=self.device).manual_seed(sampling_params.seed)
302 - else:
303 - num_seqs = len(seq_ids)
304 - selected_token_indices.extend(
305 - range(selected_token_start_idx,
306 - selected_token_start_idx + num_seqs))
307 - selected_token_start_idx += num_seqs
308 -
309 - categorized_sample_indices[
310 - sampling_params.sampling_type].extend(
311 - zip(
312 - range(
313 - categorized_sample_indices_start_idx,
314 - categorized_sample_indices_start_idx +
315 - num_seqs),
316 - range(
317 - categorized_sampled_token_indices_start_idx,
318 - categorized_sampled_token_indices_start_idx +
319 - num_seqs)))
320 - categorized_sample_indices_start_idx += num_seqs
321 - categorized_sampled_token_indices_start_idx += num_seqs
322 -
323 - if sampling_params.seed is not None:
324 - generators.append(seq_group_metadata.state.generator)
325 -
326 - selected_token_indices = torch.tensor(selected_token_indices,
327 - dtype=torch.long)
328 -
329 - categorized_sample_indices = {
330 - t: maybe_expand_dim(torch.tensor(seq_ids, dtype=torch.int), 2, 2)
331 - for t, seq_ids in categorized_sample_indices.items()
332 - }
333 -
334 - seq_data: Dict[int, SequenceData] = {}
335 - for seq_group_metadata in seq_group_metadata_list:
336 - seq_data.update(seq_group_metadata.seq_data)
337 -
338 - sampling_metadata = SamplingMetadata(
339 - seq_groups=seq_groups,
340 - seq_data=seq_data,
341 - prompt_lens=prompt_lens,
342 - selected_token_indices=selected_token_indices,
343 - categorized_sample_indices=categorized_sample_indices,
344 - generators=generators,
345 - )
346 - return sampling_metadata
347 -
256 def prepare_input_tensors(
257 self,
258 seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -364,8 +272,15 @@ class CPUModelRunner:
272 (input_tokens, input_positions,
273 attn_metadata) = self._prepare_decode(seq_group_metadata_list)
274 prompt_lens = []
367 - sampling_metadata = self._prepare_sample(seq_group_metadata_list,
368 - prompt_lens)
275 + sampling_metadata = SamplingMetadata.prepare(
276 + seq_group_metadata_list,
277 + prompt_lens,
278 + # subquery_lens is not needed if chunked prefill is not
279 + # supported. Since CPU worker doesn't support chunked prefill
280 + # just use prompt_lens instead.
281 + prompt_lens,
282 + self.device,
283 + pin_memory=False)
284 # Broadcast the metadata.
285 metadata_dict = {
286 "input_tokens": input_tokens,
@@ -389,7 +304,6 @@ class CPUModelRunner:
304 selected_token_indices=selected_token_indices,
305 categorized_sample_indices=None,
306 generators=None,
392 - perform_sampling=False,
307 )
308
309 return (input_tokens, input_positions, attn_metadata,
@@ -421,7 +335,7 @@ class CPUModelRunner:
335 logits = self.model.compute_logits(hidden_states, sampling_metadata)
336
337 # Only perform sampling in the driver worker.
424 - if not sampling_metadata.perform_sampling:
338 + if not self.is_driver_worker:
339 return None
340
341 # Sample the next token.
vllm/worker/model_runner.py
+9 -114
@@ -20,12 +20,11 @@ from vllm.lora.request import LoRARequest
20 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
21 from vllm.model_executor import SamplingMetadata
22 from vllm.model_executor.model_loader import get_model
23 -from vllm.sampling_params import SamplingParams, SamplingType
23 +from vllm.sampling_params import SamplingParams
24 from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
25 SequenceGroupMetadata)
26 -from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d, is_hip,
27 - is_pin_memory_available, make_tensor_with_pad,
28 - maybe_expand_dim)
26 +from vllm.utils import (CudaMemoryProfiler, is_hip, is_pin_memory_available,
27 + make_tensor_with_pad)
28
29 logger = init_logger(__name__)
30
@@ -547,108 +546,6 @@ class ModelRunner:
546 slot_mapping=slot_mapping,
547 )
548
550 - def _prepare_sample(
551 - self,
552 - seq_group_metadata_list: List[SequenceGroupMetadata],
553 - prompt_lens: List[int],
554 - subquery_lens: Optional[List[int]],
555 - ) -> SamplingMetadata:
556 - seq_groups: List[Tuple[List[int], SamplingParams]] = []
557 - selected_token_indices: List[int] = []
558 - generators: List[torch.Generator] = []
559 - selected_token_start_idx = 0
560 - categorized_sample_indices: Dict[SamplingType,
561 - List[Tuple[int, int]]] = {
562 - t: []
563 - for t in SamplingType
564 - }
565 - categorized_sample_indices_start_idx = 0
566 - categorized_sampled_token_indices_start_idx = 0
567 -
568 - for i, seq_group_metadata in enumerate(seq_group_metadata_list):
569 - seq_ids = list(seq_group_metadata.seq_data.keys())
570 - sampling_params = seq_group_metadata.sampling_params
571 - seq_groups.append((seq_ids, sampling_params))
572 -
573 - if seq_group_metadata.is_prompt:
574 - assert len(seq_ids) == 1
575 - assert subquery_lens is not None
576 - subquery_len = subquery_lens[i]
577 - if sampling_params.prompt_logprobs is not None:
578 - # NOTE: prompt token positions do not need sample, skip
579 - categorized_sample_indices_start_idx += subquery_len - 1
580 -
581 - categorized_sample_indices[
582 - sampling_params.sampling_type].append(
583 - (categorized_sample_indices_start_idx,
584 - categorized_sampled_token_indices_start_idx))
585 - categorized_sample_indices_start_idx += 1
586 - categorized_sampled_token_indices_start_idx += 1
587 -
588 - if sampling_params.prompt_logprobs is not None:
589 - selected_token_indices.extend(
590 - range(selected_token_start_idx,
591 - selected_token_start_idx + subquery_len - 1))
592 - selected_token_indices.append(selected_token_start_idx +
593 - subquery_len - 1)
594 - selected_token_start_idx += subquery_len
595 -
596 - if sampling_params.seed is not None:
597 - seq_group_metadata.state.generator = torch.Generator(
598 - device=self.device).manual_seed(sampling_params.seed)
599 - else:
600 - num_seqs = len(seq_ids)
601 - selected_token_indices.extend(
602 - range(selected_token_start_idx,
603 - selected_token_start_idx + num_seqs))
604 - selected_token_start_idx += num_seqs
605 -
606 - categorized_sample_indices[
607 - sampling_params.sampling_type].extend(
608 - list(
609 - zip(
610 - range(
611 - categorized_sample_indices_start_idx,
612 - categorized_sample_indices_start_idx +
613 - num_seqs),
614 - range(
615 - categorized_sampled_token_indices_start_idx,
616 - categorized_sampled_token_indices_start_idx
617 - + num_seqs))))
618 - categorized_sample_indices_start_idx += num_seqs
619 - categorized_sampled_token_indices_start_idx += num_seqs
620 -
621 - if sampling_params.seed is not None:
622 - generators.append(seq_group_metadata.state.generator)
623 -
624 - selected_token_indices = async_tensor_h2d(selected_token_indices,
625 - dtype=torch.long,
626 - target_device=self.device,
627 - pin_memory=self.pin_memory)
628 -
629 - categorized_sample_indices = {
630 - t: maybe_expand_dim(
631 - async_tensor_h2d(seq_ids,
632 - dtype=torch.int,
633 - target_device=self.device,
634 - pin_memory=self.pin_memory), 2, 2)
635 - for t, seq_ids in categorized_sample_indices.items()
636 - }
637 -
638 - seq_data: Dict[int, SequenceData] = {}
639 - for seq_group_metadata in seq_group_metadata_list:
640 - seq_data.update(seq_group_metadata.seq_data)
641 -
642 - sampling_metadata = SamplingMetadata(
643 - seq_groups=seq_groups,
644 - seq_data=seq_data,
645 - prompt_lens=prompt_lens,
646 - selected_token_indices=selected_token_indices,
647 - categorized_sample_indices=categorized_sample_indices,
648 - generators=generators,
649 - )
650 - return sampling_metadata
651 -
549 def prepare_input_tensors(
550 self,
551 seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -685,9 +582,9 @@ class ModelRunner:
582 decode_lora_requests,
583 decode_slot_mapping,
584 ) = self._prepare_decode(decode_reqs)
688 - sampling_metadata = self._prepare_sample(seq_group_metadata_list,
689 - prompt_lens,
690 - subquery_lens)
585 + sampling_metadata = SamplingMetadata.prepare(
586 + seq_group_metadata_list, prompt_lens, subquery_lens,
587 + self.device, self.pin_memory)
588
589 if not self.scheduler_config.chunked_prefill_enabled:
590 assert (len(prefill_reqs) and len(decode_reqs)) == 0
@@ -788,12 +685,9 @@ class ModelRunner:
685 **metadata_dict)
686 sampling_metadata = SamplingMetadata(
687 seq_groups=None,
791 - seq_data=None,
792 - prompt_lens=None,
688 selected_token_indices=selected_token_indices,
689 categorized_sample_indices=None,
795 - generators=None,
796 - perform_sampling=False,
690 + num_prompts=0,
691 )
692
693 # if it is a mixed batch, decode attn_metadata is broadcasted
@@ -852,7 +746,7 @@ class ModelRunner:
746 logits = self.model.compute_logits(hidden_states, sampling_metadata)
747
748 # Only perform sampling in the driver worker.
855 - if not sampling_metadata.perform_sampling:
749 + if not self.is_driver_worker:
750 return None
751
752 # Sample the next token.
@@ -860,6 +754,7 @@ class ModelRunner:
754 logits=logits,
755 sampling_metadata=sampling_metadata,
756 )
757 +
758 return output
759
760 @torch.inference_mode()
vllm/worker/neuron_model_runner.py
+12 -107
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Optional, Tuple
1 +from typing import List, Optional, Tuple
2
3 import torch
4 from torch import nn
@@ -8,10 +8,8 @@ from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
8 from vllm.logger import init_logger
9 from vllm.model_executor import SamplingMetadata
10 from vllm.model_executor.model_loader.neuron import get_neuron_model
11 -from vllm.sampling_params import SamplingParams, SamplingType
12 -from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
13 -from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
14 - make_tensor_with_pad, maybe_expand_dim)
11 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
12 +from vllm.utils import is_pin_memory_available, make_tensor_with_pad
13
14 logger = init_logger(__name__)
15
@@ -141,106 +139,6 @@ class NeuronModelRunner:
139
140 return input_tokens, input_positions, input_block_ids
141
144 - def _prepare_sample(
145 - self,
146 - seq_group_metadata_list: List[SequenceGroupMetadata],
147 - prompt_lens: List[int],
148 - ) -> SamplingMetadata:
149 - seq_groups: List[Tuple[List[int], SamplingParams]] = []
150 - selected_token_indices: List[int] = []
151 - generators: List[torch.Generator] = []
152 - selected_token_start_idx = 0
153 - categorized_sample_indices: Dict[SamplingType,
154 - List[Tuple[int, int]]] = {
155 - t: []
156 - for t in SamplingType
157 - }
158 - categorized_sample_indices_start_idx = 0
159 - categorized_sampled_token_indices_start_idx = 0
160 -
161 - for i, seq_group_metadata in enumerate(seq_group_metadata_list):
162 - seq_ids = list(seq_group_metadata.seq_data.keys())
163 - sampling_params = seq_group_metadata.sampling_params
164 - seq_groups.append((seq_ids, sampling_params))
165 -
166 - if seq_group_metadata.is_prompt:
167 - assert len(seq_ids) == 1
168 - assert prompt_lens is not None
169 - prompt_len = prompt_lens[i]
170 - if sampling_params.prompt_logprobs is not None:
171 - # NOTE: prompt token positions do not need sample, skip
172 - categorized_sample_indices_start_idx += prompt_len - 1
173 -
174 - categorized_sample_indices[
175 - sampling_params.sampling_type].append(
176 - (categorized_sample_indices_start_idx,
177 - categorized_sampled_token_indices_start_idx))
178 - categorized_sample_indices_start_idx += 1
179 - categorized_sampled_token_indices_start_idx += 1
180 -
181 - if sampling_params.prompt_logprobs is not None:
182 - selected_token_indices.extend(
183 - range(selected_token_start_idx,
184 - selected_token_start_idx + prompt_len - 1))
185 - selected_token_indices.append(selected_token_start_idx +
186 - prompt_len - 1)
187 - selected_token_start_idx += prompt_len
188 -
189 - if sampling_params.seed is not None:
190 - seq_group_metadata.state.generator = torch.Generator(
191 - device=self.device).manual_seed(sampling_params.seed)
192 - else:
193 - num_seqs = len(seq_ids)
194 - selected_token_indices.extend(
195 - range(selected_token_start_idx,
196 - selected_token_start_idx + num_seqs))
197 - selected_token_start_idx += num_seqs
198 -
199 - categorized_sample_indices[
200 - sampling_params.sampling_type].extend(
201 - zip(
202 - range(
203 - categorized_sample_indices_start_idx,
204 - categorized_sample_indices_start_idx +
205 - num_seqs),
206 - range(
207 - categorized_sampled_token_indices_start_idx,
208 - categorized_sampled_token_indices_start_idx +
209 - num_seqs)))
210 - categorized_sample_indices_start_idx += num_seqs
211 - categorized_sampled_token_indices_start_idx += num_seqs
212 -
213 - if sampling_params.seed is not None:
214 - generators.append(seq_group_metadata.state.generator)
215 -
216 - selected_token_indices = async_tensor_h2d(selected_token_indices,
217 - dtype=torch.long,
218 - target_device=self.device,
219 - pin_memory=self.pin_memory)
220 -
221 - categorized_sample_indices = {
222 - t: maybe_expand_dim(
223 - async_tensor_h2d(seq_ids,
224 - dtype=torch.int,
225 - target_device=self.device,
226 - pin_memory=self.pin_memory), 2, 2)
227 - for t, seq_ids in categorized_sample_indices.items()
228 - }
229 -
230 - seq_data: Dict[int, SequenceData] = {}
231 - for seq_group_metadata in seq_group_metadata_list:
232 - seq_data.update(seq_group_metadata.seq_data)
233 -
234 - sampling_metadata = SamplingMetadata(
235 - seq_groups=seq_groups,
236 - seq_data=seq_data,
237 - prompt_lens=prompt_lens,
238 - selected_token_indices=selected_token_indices,
239 - categorized_sample_indices=categorized_sample_indices,
240 - generators=generators,
241 - )
242 - return sampling_metadata
243 -
142 def prepare_input_tensors(
143 self,
144 seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -256,8 +154,15 @@ class NeuronModelRunner:
154 (input_tokens, input_positions,
155 input_block_ids) = self._prepare_decode(seq_group_metadata_list)
156 prompt_lens = []
259 - sampling_metadata = self._prepare_sample(seq_group_metadata_list,
260 - prompt_lens)
157 + sampling_metadata = SamplingMetadata.prepare(
158 + seq_group_metadata_list,
159 + prompt_lens,
160 + # subquery_lens is not needed if chunked prefill is not
161 + # supported. Since neuron worker doesn't support chunked prefill
162 + # just use prompt_lens instead.
163 + prompt_lens,
164 + self.device,
165 + self.pin_memory)
166
167 return (input_tokens, input_positions, input_block_ids,
168 sampling_metadata)