A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Refactoring sampler and support prompt logprob for chunked prefill (#4309)
SangBin Cho committed
Apr 26, 2024 at 22:02 UTC
603ad8481594321ceae7d54e2c0050b3638c6502
18 files changed
+862
-633
tests/samplers/test_logprobs.py
+41
-3
@@ -9,15 +9,26 @@ MODELS = ["facebook/opt-125m"]
9
10
@pytest.mark.parametrize("model", MODELS)
11
@pytest.mark.parametrize("dtype", ["half"])
12
+@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
13
+@pytest.mark.parametrize("num_top_logprobs", [6]) # 32000 == vocab_size
14
def test_get_prompt_logprobs(
15
hf_runner,
16
vllm_runner,
17
model,
18
dtype,
19
+ chunked_prefill_token_size: int,
20
+ num_top_logprobs: int,
21
example_prompts,
22
):
23
+ max_num_seqs = 256
24
+ enable_chunked_prefill = False
25
+ max_num_batched_tokens = None
26
+ if chunked_prefill_token_size != -1:
27
+ enable_chunked_prefill = True
28
+ max_num_seqs = min(chunked_prefill_token_size, max_num_seqs)
29
+ max_num_batched_tokens = chunked_prefill_token_size
30
+
31
max_tokens = 5
20
- num_top_logprobs = 6
32
hf_model = hf_runner(model, dtype=dtype)
33
hf_logprobs = hf_model.generate_greedy_logprobs(
34
example_prompts,
@@ -25,10 +36,17 @@ def test_get_prompt_logprobs(
36
)
37
del hf_model
38
28
- vllm_model = vllm_runner(model, dtype=dtype, max_logprobs=num_top_logprobs)
39
+ vllm_model = vllm_runner(
40
+ model,
41
+ dtype=dtype,
42
+ max_logprobs=num_top_logprobs,
43
+ enable_chunked_prefill=enable_chunked_prefill,
44
+ max_num_batched_tokens=max_num_batched_tokens,
45
+ max_num_seqs=max_num_seqs,
46
+ )
47
vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
48
logprobs=num_top_logprobs,
31
- prompt_logprobs=5,
49
+ prompt_logprobs=num_top_logprobs,
50
temperature=0.0)
51
vllm_results = vllm_model.model.generate(
52
example_prompts, sampling_params=vllm_sampling_params)
@@ -52,9 +70,18 @@ def test_get_prompt_logprobs(
70
"The output text from the top logprob for each token position "
71
"should be the same as the output text in the result.")
72
73
+ # The first prompt logprob is always None
74
+ assert result.prompt_logprobs[0] is None
75
+ for prompt_logprobs in result.prompt_logprobs[1:]:
76
+ # If the prompt token is not included in the top X
77
+ # logprob, it can return 1 more data
78
+ assert (len(prompt_logprobs) == num_top_logprobs
79
+ or len(prompt_logprobs) == num_top_logprobs + 1)
80
+
81
# Test whether prompt logprobs are consistent with HF
82
for vllm_result, hf_logprob in zip(vllm_results, hf_logprobs):
83
# Check prompt logprobs
84
+ # The first prompt logprob is always None, so we compare it from 1:.
85
vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
86
for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
87
for token_id, logprob in vllm_prompt_logprob_dict.items():
@@ -74,6 +101,17 @@ def test_get_prompt_logprobs(
101
"The token should be decoded by the time it is returned "
102
" to the user.")
103
104
+ # Test if prompt logprobs are correctly set.
105
+ for vllm_result in vllm_results:
106
+ token_ids = vllm_result.prompt_token_ids
107
+ prompt_logprobs = vllm_result.prompt_logprobs
108
+
109
+ # The first token doesn't have logprob.
110
+ assert prompt_logprobs[0] is None
111
+
112
+ for token_id, logprob_dict in zip(token_ids[1:], prompt_logprobs[1:]):
113
+ assert token_id in logprob_dict
114
+
115
116
def test_max_logprobs():
117
runner = VllmRunner("facebook/opt-125m", max_logprobs=1)
tests/samplers/test_sampler.py
+31
-16
@@ -8,6 +8,7 @@ import torch
8
from transformers import GenerationConfig, GenerationMixin
9
10
from vllm.model_executor.layers.sampler import Sampler
11
+from vllm.model_executor.sampling_metadata import SamplingMetadata
12
from vllm.model_executor.utils import set_random_seed
13
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
14
from vllm.utils import Counter
@@ -54,6 +55,7 @@ def _do_sample(
55
sampler: MockLogitsSampler,
56
model_runner: ModelRunner,
57
sampling_params: SamplingParams,
58
+ device: str,
59
):
60
seq_group_metadata_list = []
61
prompt_lens = []
@@ -68,9 +70,12 @@ def _do_sample(
70
))
71
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
72
71
- sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
72
- prompt_lens,
73
- subquery_lens=prompt_lens)
73
+ sampling_metadata = SamplingMetadata.prepare(
74
+ seq_group_metadata_list,
75
+ prompt_lens,
76
+ subquery_lens=prompt_lens,
77
+ device=device,
78
+ pin_memory=model_runner.pin_memory)
79
return sampler(logits=input_tensor, sampling_metadata=sampling_metadata)
80
81
@@ -85,7 +90,7 @@ def test_sampler_all_greedy(seed: int, device: str):
90
91
sampling_params = SamplingParams(temperature=0)
92
sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
88
- sampling_params)
93
+ sampling_params, device)
94
expected = torch.argmax(fake_logits, dim=-1)
95
for i, sequence_output in enumerate(sampler_output):
96
for nth_output in sequence_output.samples:
@@ -111,7 +116,7 @@ def test_sampler_all_random(seed: int, device: str):
116
n=random.randint(1, 10),
117
)
118
sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
114
- sampling_params)
119
+ sampling_params, device)
120
121
for i, sequence_output in enumerate(sampler_output):
122
for nth_output in sequence_output.samples:
@@ -137,7 +142,7 @@ def test_sampler_all_random_seed(seed: int, device: str):
142
seed=random.randint(0, 10000),
143
)
144
sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
140
- sampling_params)
145
+ sampling_params, device)
146
147
for i, sequence_output in enumerate(sampler_output):
148
for nth_output in sequence_output.samples:
@@ -160,10 +165,10 @@ def test_sampler_all_random_seed_deterministic(seed: int, device: str):
165
seed=random.randint(0, 10000),
166
)
167
first_sampler_output = _do_sample(batch_size, fake_logits, sampler,
163
- model_runner, sampling_params)
168
+ model_runner, sampling_params, device)
169
170
second_sampler_output = _do_sample(batch_size, fake_logits, sampler,
166
- model_runner, sampling_params)
171
+ model_runner, sampling_params, device)
172
173
assert first_sampler_output == second_sampler_output
174
@@ -183,7 +188,8 @@ def test_sampler_all_beam(seed: int, device: str):
188
best_of=2,
189
use_beam_search=True,
190
)
186
- _do_sample(batch_size, fake_logits, sampler, model_runner, sampling_params)
191
+ _do_sample(batch_size, fake_logits, sampler, model_runner, sampling_params,
192
+ device)
193
# no assertion here as I am not sure how to determine whether
194
# the outputs are expected - in other words, this just tests
195
# whether there are no exceptions in the sampler
@@ -443,10 +449,12 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
449
"batch size")
450
451
_, fake_logits, sampler, model_runner = _prepare_test(batch_size)
446
- sampling_metadata = model_runner._prepare_sample(
452
+ sampling_metadata = SamplingMetadata.prepare(
453
seq_group_metadata_list,
454
prompt_lens=prompt_lens if prompt_lens else None,
449
- subquery_lens=prompt_lens if prompt_lens else None)
455
+ subquery_lens=prompt_lens if prompt_lens else None,
456
+ device=device,
457
+ pin_memory=model_runner.pin_memory)
458
# the logits tensor is modified in-place by the sampler
459
_ = sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
460
@@ -530,8 +538,12 @@ def test_sampler_mixed(seed: int, device: str):
538
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
539
540
def test_sampling(model_runner: ModelRunner):
533
- sampling_metadata = model_runner._prepare_sample(
534
- seq_group_metadata_list, prompt_lens, subquery_lens=prompt_lens)
541
+ sampling_metadata = SamplingMetadata.prepare(
542
+ seq_group_metadata_list,
543
+ prompt_lens,
544
+ subquery_lens=prompt_lens,
545
+ device=device,
546
+ pin_memory=model_runner.pin_memory)
547
sampler_output = sampler(logits=fake_logits,
548
sampling_metadata=sampling_metadata)
549
@@ -627,9 +639,12 @@ def test_sampler_top_k_top_p(seed: int, device: str):
639
))
640
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
641
630
- sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
631
- prompt_lens,
632
- subquery_lens=prompt_lens)
642
+ sampling_metadata = SamplingMetadata.prepare(
643
+ seq_group_metadata_list,
644
+ prompt_lens,
645
+ subquery_lens=prompt_lens,
646
+ device=device,
647
+ pin_memory=model_runner.pin_memory)
648
649
sample_probs = None
650
tests/test_logits_processor.py
+7
-3
@@ -6,6 +6,7 @@ import pytest
6
import torch
7
8
from vllm.model_executor.layers.logits_processor import LogitsProcessor
9
+from vllm.model_executor.sampling_metadata import SamplingMetadata
10
from vllm.model_executor.utils import set_random_seed
11
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12
from vllm.worker.model_runner import ModelRunner
@@ -82,9 +83,12 @@ def test_logits_processors(seed: int, device: str):
83
))
84
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
85
85
- sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
86
- prompt_lens,
87
- subquery_lens=prompt_lens)
86
+ sampling_metadata = SamplingMetadata.prepare(
87
+ seq_group_metadata_list,
88
+ prompt_lens,
89
+ subquery_lens=prompt_lens,
90
+ device=model_runner.device,
91
+ pin_memory=model_runner.pin_memory)
92
logits_processor_output = logits_processor(
93
embedding=None,
94
hidden_states=input_tensor,
tests/worker/test_model_runner.py
+13
-6
@@ -2,6 +2,7 @@ import pytest
2
import torch
3
4
from vllm.config import ModelConfig, SchedulerConfig
5
+from vllm.model_executor.sampling_metadata import SamplingMetadata
6
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
7
from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
8
@@ -97,9 +98,12 @@ def test_prepare_prompt(batch_size):
98
assert len(input_positions) == sum(prompt_lens)
99
torch.testing.assert_close(input_tokens, input_positions)
100
100
- sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
101
- prompt_lens,
102
- subquery_lens=prompt_lens)
101
+ sampling_metadata = SamplingMetadata.prepare(
102
+ seq_group_metadata_list,
103
+ prompt_lens,
104
+ subquery_lens=prompt_lens,
105
+ device=model_runner.device,
106
+ pin_memory=model_runner.pin_memory)
107
assert len(input_tokens) == sum(prompt_lens)
108
assert len(input_positions) == sum(prompt_lens)
109
actual = sampling_metadata.selected_token_indices
@@ -195,9 +199,12 @@ def test_prepare_decode_cuda_graph(batch_size):
199
for prompt_len in prompt_lens:
200
expected_selected_token_indices.append(selected_token_start_idx)
201
selected_token_start_idx += 1
198
- sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
199
- prompt_lens,
200
- subquery_lens=prompt_lens)
202
+ sampling_metadata = SamplingMetadata.prepare(
203
+ seq_group_metadata_list,
204
+ prompt_lens,
205
+ subquery_lens=prompt_lens,
206
+ device=model_runner.device,
207
+ pin_memory=model_runner.pin_memory)
208
actual = sampling_metadata.selected_token_indices
209
expected = torch.tensor(expected_selected_token_indices,
210
device=actual.device,
vllm/core/scheduler.py
+15
@@ -915,6 +915,20 @@ class Scheduler:
915
self.block_manager.get_common_computed_block_ids(
916
seq_group.get_seqs(status=SequenceStatus.RUNNING)))
917
918
+ do_sample = True
919
+ if seq_group.is_prefill():
920
+ seqs = seq_group.get_seqs()
921
+ # Prefill has only 1 sequence.
922
+ assert len(seqs) == 1
923
+ # In the next iteration, all prompt tokens are not computed.
924
+ # It means the prefill is chunked, and we don't need sampling.
925
+ # NOTE: We use get_len instead of get_prompt_len because when
926
+ # a sequence is preempted, prefill includes previous generated
927
+ # output tokens.
928
+ if (token_chunk_size + seqs[0].data.get_num_computed_tokens() <
929
+ seqs[0].data.get_len()):
930
+ do_sample = False
931
+
932
# It assumes the scheduled_seq_groups is ordered by
933
# prefill < decoding.
934
is_prompt = seq_group.is_prefill()
@@ -924,6 +938,7 @@ class Scheduler:
938
seq_data=seq_data,
939
sampling_params=seq_group.sampling_params,
940
block_tables=block_tables,
941
+ do_sample=do_sample,
942
token_chunk_size=token_chunk_size,
943
lora_request=seq_group.lora_request,
944
computed_block_nums=common_computed_block_nums,
vllm/engine/async_llm_engine.py
+1
-1
@@ -219,7 +219,7 @@ class _AsyncLLMEngine(LLMEngine):
219
220
request_outputs = self._process_model_outputs(
221
output, scheduler_outputs.scheduled_seq_groups,
222
- scheduler_outputs.ignored_seq_groups)
222
+ scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
223
224
# Log stats.
225
if self.log_stats:
vllm/engine/llm_engine.py
+13
-12
@@ -22,7 +22,7 @@ from vllm.lora.request import LoRARequest
22
from vllm.outputs import RequestOutput
23
from vllm.sampling_params import SamplingParams
24
from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
25
- SequenceGroup, SequenceStage)
25
+ SequenceGroup, SequenceGroupMetadata)
26
from vllm.transformers_utils.detokenizer import Detokenizer
27
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
28
get_tokenizer_group)
@@ -476,9 +476,12 @@ class LLMEngine:
476
return self.scheduler.has_unfinished_seqs()
477
478
def _process_model_outputs(
479
- self, output: List[SamplerOutput],
480
- scheduled_seq_groups: List[SequenceGroup],
481
- ignored_seq_groups: List[SequenceGroup]) -> List[RequestOutput]:
479
+ self,
480
+ output: List[SamplerOutput],
481
+ scheduled_seq_groups: List[SequenceGroup],
482
+ ignored_seq_groups: List[SequenceGroup],
483
+ seq_group_metadata_list: List[SequenceGroupMetadata],
484
+ ) -> List[RequestOutput]:
485
"""Apply the model output to the sequences in the scheduled seq groups.
486
487
Returns RequestOutputs that can be returned to the client.
@@ -492,17 +495,15 @@ class LLMEngine:
495
sampler_outputs=output, num_seq_groups=len(scheduled_seq_groups))
496
497
# Update the scheduled sequence groups with the model outputs.
495
- for scheduled_seq_group, outputs in zip(scheduled_seq_groups,
496
- output_by_sequence_group):
498
+ for scheduled_seq_group, outputs, seq_group_meta in zip(
499
+ scheduled_seq_groups, output_by_sequence_group,
500
+ seq_group_metadata_list):
501
seq_group = scheduled_seq_group.seq_group
502
seq_group.update_num_computed_tokens(
503
scheduled_seq_group.token_chunk_size)
504
501
- # If all sequences in the sequence group are in DECODE, then we can
502
- # process the output tokens. Otherwise, they are (chunked) prefill
503
- # samples and should not be processed.
504
- stages = [seq.data._stage for seq in seq_group.seqs_dict.values()]
505
- if all(stage == SequenceStage.DECODE for stage in stages):
505
+ self.output_processor.process_prompt_logprob(seq_group, outputs)
506
+ if seq_group_meta.do_sample:
507
self.output_processor.process_outputs(seq_group, outputs)
508
509
# Free the finished sequence groups.
@@ -585,7 +586,7 @@ class LLMEngine:
586
587
request_outputs = self._process_model_outputs(
588
output, scheduler_outputs.scheduled_seq_groups,
588
- scheduler_outputs.ignored_seq_groups)
589
+ scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
590
591
# Log stats.
592
if self.log_stats:
vllm/engine/output_processor/interfaces.py
+6
@@ -68,3 +68,9 @@ class SequenceGroupOutputProcessor(ABC):
68
scheduler.
69
"""
70
pass
71
+
72
+ @abstractmethod
73
+ def process_prompt_logprob(self, seq_group: SequenceGroup,
74
+ outputs: List[SequenceGroupOutput]) -> None:
75
+ """Update prompt logprobs received from outputs to seq_group."""
76
+ pass
vllm/engine/output_processor/multi_step.py
+9
@@ -44,6 +44,15 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
44
self.get_tokenizer_for_seq = get_tokenizer_for_seq
45
self.stop_checker = stop_checker
46
47
+ def process_prompt_logprob(self, seq_group: SequenceGroup,
48
+ outputs: List[SequenceGroupOutput]) -> None:
49
+ # TODO(sang): Prompt logprob currently not implemented in multi step
50
+ # workers.
51
+ logger.warning(
52
+ "Prompt logprob is not supported by multi step workers. "
53
+ "(e.g., speculative decode uses multi step workers).")
54
+ pass
55
+
56
def process_outputs(self, sequence_group: SequenceGroup,
57
outputs: List[SequenceGroupOutput]) -> None:
58
"""Append new tokens in the outputs to sequences in the sequence group.
vllm/engine/output_processor/single_step.py
+14
-8
@@ -55,17 +55,23 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
55
), f"{type(self)} does not support multiple outputs per step"
56
return self._process_sequence_group_outputs(sequence_group, outputs[0])
57
58
- def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
59
- outputs: SequenceGroupOutput) -> None:
60
-
61
- # Process prompt logprobs
62
- prompt_logprobs = outputs.prompt_logprobs
63
- if prompt_logprobs is not None and \
64
- seq_group.sampling_params.detokenize and self.detokenizer:
58
+ def process_prompt_logprob(self, seq_group: SequenceGroup,
59
+ outputs: List[SequenceGroupOutput]) -> None:
60
+ assert len(outputs) == 1, ("Single step should only has 1 output.")
61
+ output = outputs[0]
62
+ prompt_logprobs = output.prompt_logprobs
63
+ if (prompt_logprobs is not None
64
+ and seq_group.sampling_params.detokenize and self.detokenizer):
65
self.detokenizer.decode_prompt_logprobs_inplace(
66
seq_group, prompt_logprobs)
67
- seq_group.prompt_logprobs = prompt_logprobs
67
+ if not seq_group.prompt_logprobs:
68
+ # The first prompt token's logprob is None because it doesn't
69
+ # have tokens that are precedent.
70
+ seq_group.prompt_logprobs = [None]
71
+ seq_group.prompt_logprobs.extend(prompt_logprobs)
72
73
+ def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
74
+ outputs: SequenceGroupOutput) -> None:
75
# Process samples
76
samples = outputs.samples
77
parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
vllm/engine/output_processor/util.py
+4
-3
@@ -1,10 +1,11 @@
1
from typing import List
2
3
-from vllm.sequence import SamplerOutput
3
+from vllm.sequence import SamplerOutput, SequenceGroupOutput
4
5
6
-def create_output_by_sequence_group(sampler_outputs: List[SamplerOutput],
7
- num_seq_groups: int):
6
+def create_output_by_sequence_group(
7
+ sampler_outputs: List[SamplerOutput],
8
+ num_seq_groups: int) -> List[List[SequenceGroupOutput]]:
9
"""Helper method which transforms a 2d list organized by
10
[step][sequence group] into [sequence group][step].
11
"""
vllm/model_executor/layers/logits_processor.py
+12
-15
@@ -83,30 +83,27 @@ def _apply_logits_processors(
83
logits: torch.Tensor,
84
sampling_metadata: SamplingMetadata,
85
) -> torch.Tensor:
86
- logits_row_idx = 0
86
found_logits_processors = False
88
- for i, seq_group in enumerate(sampling_metadata.seq_groups):
89
- seq_ids, sampling_params = seq_group
87
+ logits_processed = 0
88
+ for seq_group in sampling_metadata.seq_groups:
89
+ seq_ids = seq_group.seq_ids
90
+ sampling_params = seq_group.sampling_params
91
logits_processors = sampling_params.logits_processors
91
- # handle prompt_logprobs by skipping rows in logits added for
92
- # the prompt tokens (prompt logprobs are not processed)
93
- if (i < sampling_metadata.num_prompts
94
- and sampling_params.prompt_logprobs is not None):
95
- assert len(seq_ids) == 1
96
- logits_row_idx += sampling_metadata.prompt_lens[i] - 1
92
93
if logits_processors:
94
found_logits_processors = True
100
- for seq_id in seq_ids:
95
+ for seq_id, logits_row_idx in zip(seq_ids,
96
+ seq_group.sample_indices):
97
logits_row = logits[logits_row_idx]
102
- token_ids = sampling_metadata.seq_data[seq_id].output_token_ids
98
+ token_ids = seq_group.seq_data[seq_id].output_token_ids
99
for logits_processor in logits_processors:
100
logits_row = logits_processor(token_ids, logits_row)
101
logits[logits_row_idx] = logits_row
106
- logits_row_idx += 1
107
- else:
108
- logits_row_idx += len(seq_ids)
102
+
103
+ logits_processed += len(seq_group.sample_indices) + len(
104
+ seq_group.prompt_logprob_indices)
105
+
106
if found_logits_processors:
107
# verifies that no rows in logits were missed unexpectedly
111
- assert logits_row_idx == logits.shape[0]
108
+ assert logits_processed == logits.shape[0]
109
return logits
vllm/model_executor/layers/sampler.py
+366
-178
@@ -7,11 +7,11 @@ import torch.nn as nn
7
8
from vllm.model_executor.layers.ops.sample import sample as sample_triton
9
from vllm.model_executor.sampling_metadata import (SamplingMetadata,
10
- SamplingTensors)
11
-from vllm.sampling_params import SamplingParams, SamplingType
10
+ SamplingTensors,
11
+ SequenceGroupToSample)
12
+from vllm.sampling_params import SamplingType
13
from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
13
- SamplerOutput, SequenceData, SequenceGroupOutput,
14
- SequenceOutput)
14
+ SamplerOutput, SequenceGroupOutput, SequenceOutput)
15
16
17
class Sampler(nn.Module):
@@ -48,11 +48,14 @@ class Sampler(nn.Module):
48
logits: torch.Tensor,
49
sampling_metadata: SamplingMetadata,
50
) -> Optional[SamplerOutput]:
51
+ """
52
+ Args:
53
+ logits: (num_tokens, vocab_size).
54
+ sampling_metadata: Metadata for sampling.
55
+ """
56
assert logits is not None
57
_, vocab_size = logits.shape
58
54
- # Apply min_tokens penalty which sets stop tokens to -inf if min_tokens
55
- # have not been generated yet
59
logits = _apply_min_tokens_penalty(logits, sampling_metadata)
60
61
# Prepare sampling tensors with pinned memory to avoid blocking.
@@ -83,7 +86,6 @@ class Sampler(nn.Module):
86
# Compute the probabilities.
87
probs = torch.softmax(logits, dim=-1, dtype=torch.float)
88
# Compute the log probabilities.
86
- # Use log_softmax to ensure numerical stability.
89
logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
90
91
# Sample the next tokens.
@@ -149,24 +151,28 @@ def _apply_min_tokens_penalty(
151
logits: torch.Tensor,
152
sampling_metadata: SamplingMetadata,
153
) -> torch.Tensor:
154
+ """Apply min_tokens penalty which sets stop tokens to -inf if min_tokens
155
+ have not been generated yet
156
+ """
157
# list of indices in logits that will be set to -inf
158
logits_to_penalize = []
154
- start_idx = 0
155
- for i, seq_group in enumerate(sampling_metadata.seq_groups):
156
- seq_ids, sampling_params = seq_group
157
-
158
- # handle prompt_logprobs by skipping rows in logits added for the prompt
159
- # tokens (prompt logprobs are not penalized)
160
- if (i < sampling_metadata.num_prompts
161
- and sampling_params.prompt_logprobs is not None):
162
- assert len(seq_ids) == 1
163
- start_idx += sampling_metadata.prompt_lens[i] - 1
159
+ logits_applied = 0
160
+ for seq_group in sampling_metadata.seq_groups:
161
+ seq_ids = seq_group.seq_ids
162
+ sampling_params = seq_group.sampling_params
163
+
164
+ sample_indices = seq_group.sample_indices
165
+ logits_applied += len(sample_indices) + len(
166
+ seq_group.prompt_logprob_indices)
167
+ if not seq_group.do_sample:
168
+ continue
169
170
+ start_idx = sample_indices[0]
171
min_tokens = sampling_params.min_tokens
172
if min_tokens > 0:
173
seqs_to_penalize = []
174
for i, seq_id in enumerate(seq_ids):
169
- seq_data = sampling_metadata.seq_data[seq_id]
175
+ seq_data = seq_group.seq_data[seq_id]
176
if len(seq_data.output_token_ids) < min_tokens:
177
seqs_to_penalize.append(i)
178
@@ -180,15 +186,13 @@ def _apply_min_tokens_penalty(
186
logits_to_penalize.extend(
187
itertools.product(seqs_to_penalize, token_ids_to_penalize))
188
183
- start_idx += len(seq_ids)
184
-
189
if logits_to_penalize:
190
# use zip and * to group indices along each dimension
191
# eg. [ (1,2), (1,3), (5,6) ] -> ( (1,1,5), (2,3,6) )
192
logits[tuple(zip(*logits_to_penalize))] = -float("inf")
193
194
# verifies that no rows in logits were missed unexpectedly
191
- assert start_idx == logits.shape[0]
195
+ assert logits_applied == logits.shape[0]
196
return logits
197
198
@@ -265,14 +269,30 @@ def _apply_min_p(
269
270
271
def _greedy_sample(
268
- selected_seq_groups: List[Tuple[List[int], SamplingParams]],
272
+ selected_seq_groups: List[SequenceGroupToSample],
273
samples: torch.Tensor,
274
) -> List[Tuple[List[int], List[int]]]:
275
+ """Run greedy sampling on a given samples.
276
+
277
+ Args:
278
+ selected_seq_groups: A list of sequence groups batched.
279
+ samples: (num_selected_samples,) A tensor of samples. The length of
280
+ samples could be smaller than selected_seq_groups if
281
+ seq_group.do_sample is False.
282
+ Returns:
283
+ Tuple of (next_token_ids, parent_ids). The length of returned list is
284
+ same as the length of selected_seq_groups. If the corresponding
285
+ seq_group has do_sample=False, tuple contains ([], [])
286
+ """
287
samples = samples.tolist()
288
sample_idx = 0
289
results = []
290
for seq_group in selected_seq_groups:
275
- seq_ids, _ = seq_group
291
+ if not seq_group.do_sample:
292
+ results.append(([], []))
293
+ continue
294
+
295
+ seq_ids = seq_group.seq_ids
296
num_parent_seqs = len(seq_ids)
297
assert num_parent_seqs == 1, (
298
"Greedy sampling should have only one seq.")
@@ -284,16 +304,33 @@ def _greedy_sample(
304
305
306
def _random_sample(
287
- selected_seq_groups: List[Tuple[List[int], SamplingParams]],
288
- is_prompts: List[bool],
307
+ selected_seq_groups: List[SequenceGroupToSample],
308
random_samples: torch.Tensor,
309
) -> List[Tuple[List[int], List[int]]]:
310
+ """Run random sampling on a given samples.
311
+
312
+ Args:
313
+ selected_seq_groups: A list of sequence groups batched.
314
+ random_samples: (num_selected_samples,) A tensor of samples. The
315
+ length of samples could be smaller than selected_seq_groups if
316
+ seq_group.do_sample is False.
317
+ Returns:
318
+ Tuple of (next_token_ids, parent_ids). The length of returned list is
319
+ same as the length of selected_seq_groups. If the corresponding
320
+ seq_group has do_sample=False, tuple contains ([], [])
321
+ """
322
# Find the maximum best_of value of the prompt phase requests.
323
random_samples = random_samples.cpu()
324
sample_idx = 0
325
results = []
295
- for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
296
- seq_ids, sampling_params = seq_group
326
+ for seq_group in selected_seq_groups:
327
+ if not seq_group.do_sample:
328
+ results.append(([], []))
329
+ continue
330
+
331
+ seq_ids = seq_group.seq_ids
332
+ sampling_params = seq_group.sampling_params
333
+ is_prompt = seq_group.is_prompt
334
num_parent_seqs = len(seq_ids)
335
if is_prompt:
336
# Prompt phase.
@@ -311,11 +348,20 @@ def _random_sample(
348
349
350
def _beam_search_sample(
314
- selected_seq_groups: List[Tuple[List[int], SamplingParams]],
315
- is_prompts: List[bool],
316
- seq_data: Dict[int, SequenceData],
351
+ selected_seq_groups: List[SequenceGroupToSample],
352
logprobs: torch.Tensor,
353
) -> List[Tuple[List[int], List[int]]]:
354
+ """Run beam sampling on a given samples.
355
+
356
+ Args:
357
+ selected_seq_groups: A list of sequence groups batched.
358
+ logprobs: (num_selected_samples, vocab_size,) A tensor of logprob
359
+ on selected sample indices.
360
+ Returns:
361
+ Tuple of (next_token_ids, parent_ids). The length of returned list is
362
+ same as the length of selected_seq_groups. If the corresponding
363
+ seq_group has do_sample=False, tuple contains ([], [])
364
+ """
365
# We sample 2 * beam_width candidates to make sure that with high
366
# probability we can get `beam_width` candidates in addition to
367
# the finished sequences for the next iteration. See
@@ -327,8 +373,13 @@ def _beam_search_sample(
373
# other sampling methods.
374
sample_idx = 0
375
results = []
330
- for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
331
- seq_ids, sampling_params = seq_group
376
+ for seq_group in selected_seq_groups:
377
+ if not seq_group.do_sample:
378
+ results.append(([], []))
379
+ continue
380
+
381
+ is_prompt = seq_group.is_prompt
382
+ seq_ids, sampling_params = seq_group.seq_ids, seq_group.sampling_params
383
num_parent_seqs = len(seq_ids)
384
beam_width = sampling_params.best_of
385
seq_group_logprobs = logprobs[sample_idx:sample_idx + num_parent_seqs]
@@ -343,7 +394,8 @@ def _beam_search_sample(
394
else:
395
# Generation phase.
396
cumulative_logprobs = [
346
- seq_data[seq_id].cumulative_logprob for seq_id in seq_ids
397
+ seq_group.seq_data[seq_id].cumulative_logprob
398
+ for seq_id in seq_ids
399
]
400
cumulative_logprobs = torch.tensor(
401
cumulative_logprobs,
@@ -371,8 +423,7 @@ def _beam_search_sample(
423
def _multinomial(
424
probs: torch.Tensor,
425
num_samples: int,
374
- seq_groups: Optional[List[Tuple[List[int], SamplingParams]]] = None,
375
- generators: Optional[List[torch.Generator]] = None,
426
+ seq_groups: Optional[List[SequenceGroupToSample]] = None,
427
) -> torch.Tensor:
428
if num_samples > 1:
429
# This is equivalent to torch.repeat_interleaved (which also
@@ -388,9 +439,11 @@ def _multinomial(
439
q.exponential_()
440
else:
441
sample_idx = 0
391
- for (seq_ids, _), generator in zip(seq_groups, generators):
442
+ for seq_group in seq_groups:
443
+ seq_ids = seq_group.seq_ids
444
next_sample_idx = sample_idx + len(seq_ids) * num_samples
393
- q[sample_idx:next_sample_idx].exponential_(generator=generator)
445
+ q[sample_idx:next_sample_idx].exponential_(
446
+ generator=seq_group.generator)
447
sample_idx = next_sample_idx
448
return probs.div_(q).argmax(dim=1).view(-1, num_samples)
449
@@ -405,7 +458,7 @@ def _sample_with_torch(
458
categorized_seq_group_ids = {t: [] for t in SamplingType}
459
categorized_sample_indices = sampling_metadata.categorized_sample_indices
460
for i, seq_group in enumerate(sampling_metadata.seq_groups):
408
- _, sampling_params = seq_group
461
+ sampling_params = seq_group.sampling_params
462
sampling_type = sampling_params.sampling_type
463
categorized_seq_group_ids[sampling_type].append(i)
464
@@ -429,13 +482,11 @@ def _sample_with_torch(
482
num_tokens = len(sample_indices)
483
if num_tokens == 0:
484
continue
432
- seq_group_ids = categorized_seq_group_ids[sampling_type]
433
- seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
434
- is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
435
- sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
436
- is_prompts, sample_indices)
437
- long_sample_indices = sample_indices.long()
485
486
+ seq_group_id = categorized_seq_group_ids[sampling_type]
487
+ seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_id]
488
+ sample_metadata[sampling_type] = (seq_group_id, seq_groups)
489
+ long_sample_indices = sample_indices.long()
490
if sampling_type == SamplingType.GREEDY:
491
greedy_samples = torch.argmax(logprobs[long_sample_indices],
492
dim=-1)
@@ -455,14 +506,13 @@ def _sample_with_torch(
506
507
elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
508
max_best_of_in_batch = 1
458
- for seq_group, is_prompt in zip(seq_groups, is_prompts):
459
- if is_prompt:
460
- _, sampling_params = seq_group
509
+ for seq_group in seq_groups:
510
+ if seq_group.is_prompt:
511
+ sampling_params = seq_group.sampling_params
512
max_best_of_in_batch = max(max_best_of_in_batch,
513
sampling_params.best_of)
514
seeded_args = {} if sampling_type == SamplingType.RANDOM else {
515
"seq_groups": seq_groups,
465
- "generators": sampling_metadata.generators,
516
}
517
518
multinomial_samples[sampling_type] = _multinomial(
@@ -481,25 +531,22 @@ def _sample_with_torch(
531
532
# GPU<->CPU sync happens in the loop below.
533
# This also converts the sample output to Python objects.
484
-
534
for sampling_type in SamplingType:
535
if sampling_type not in sample_metadata:
536
continue
488
- seq_group_ids, seq_groups, is_prompts, sample_indices = sample_metadata[
489
- sampling_type]
537
+ (seq_group_id, seq_groups) = sample_metadata[sampling_type]
538
if sampling_type == SamplingType.GREEDY:
539
sample_results = _greedy_sample(seq_groups, greedy_samples)
540
elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
493
- sample_results = _random_sample(seq_groups, is_prompts,
541
+ sample_results = _random_sample(seq_groups,
542
multinomial_samples[sampling_type])
543
elif sampling_type == SamplingType.BEAM:
496
- sample_results = _beam_search_sample(seq_groups, is_prompts,
497
- sampling_metadata.seq_data,
544
+ sample_results = _beam_search_sample(seq_groups,
545
beam_search_logprobs)
499
- sample_results_dict.update(zip(seq_group_ids, sample_results))
546
+ sample_results_dict.update(zip(seq_group_id, sample_results))
547
548
sample_results = [
502
- sample_results_dict[i]
549
+ sample_results_dict.get(i, ([], []))
550
for i in range(len(sampling_metadata.seq_groups))
551
]
552
return sample_results, sampled_token_ids_tensor
@@ -514,7 +561,7 @@ def _sample_with_triton_kernel(
561
categorized_seq_group_ids = {t: [] for t in SamplingType}
562
categorized_sample_indices = sampling_metadata.categorized_sample_indices
563
for i, seq_group in enumerate(sampling_metadata.seq_groups):
517
- _, sampling_params = seq_group
564
+ sampling_params = seq_group.sampling_params
565
sampling_type = sampling_params.sampling_type
566
categorized_seq_group_ids[sampling_type].append(i)
567
@@ -530,17 +577,16 @@ def _sample_with_triton_kernel(
577
num_tokens = len(sample_indices)
578
if num_tokens == 0:
579
continue
533
- seq_group_ids = categorized_seq_group_ids[sampling_type]
534
- seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
535
- is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
536
- sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
537
- is_prompts, sample_indices,
580
+ seq_group_id = categorized_seq_group_ids[sampling_type]
581
+ seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_id]
582
+ sample_metadata[sampling_type] = (seq_group_id, seq_groups,
583
+ sample_indices,
584
sampled_token_indices)
585
if sampling_type in (SamplingType.GREEDY, SamplingType.RANDOM,
586
SamplingType.RANDOM_SEED):
541
- for seq_group, is_prompt in zip(seq_groups, is_prompts):
542
- if is_prompt:
543
- _, sampling_params = seq_group
587
+ for seq_group in seq_groups:
588
+ if seq_group.is_prompt:
589
+ sampling_params = seq_group.sampling_params
590
max_best_of_in_batch = max(max_best_of_in_batch,
591
sampling_params.best_of)
592
elif sampling_type == SamplingType.BEAM:
@@ -564,22 +610,21 @@ def _sample_with_triton_kernel(
610
for sampling_type in SamplingType:
611
if sampling_type not in sample_metadata:
612
continue
567
- (seq_group_ids, seq_groups, is_prompts, sample_indices,
613
+ (seq_group_id, seq_groups, sample_indices,
614
sampled_token_indices) = sample_metadata[sampling_type]
615
if sampling_type == SamplingType.GREEDY:
616
sample_results = _greedy_sample(
617
seq_groups, sampled_tokens[sampled_token_indices][:, 0])
618
elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
619
sample_results = _random_sample(
574
- seq_groups, is_prompts, sampled_tokens[sampled_token_indices])
620
+ seq_groups, sampled_tokens[sampled_token_indices])
621
elif sampling_type == SamplingType.BEAM:
576
- sample_results = _beam_search_sample(seq_groups, is_prompts,
577
- sampling_metadata.seq_data,
622
+ sample_results = _beam_search_sample(seq_groups,
623
beam_search_logprobs)
579
- sample_results_dict.update(zip(seq_group_ids, sample_results))
624
+ sample_results_dict.update(zip(seq_group_id, sample_results))
625
626
sample_results = [
582
- sample_results_dict[i]
627
+ sample_results_dict.get(i, ([], []))
628
for i in range(len(sampling_metadata.seq_groups))
629
]
630
return sample_results
@@ -590,6 +635,18 @@ def _sample(
635
sampling_metadata: SamplingMetadata, sampling_tensors: SamplingTensors,
636
include_gpu_probs_tensor: bool, modify_greedy_probs: bool
637
) -> Tuple[List[Tuple[List[int], List[int]]], Optional[torch.Tensor]]:
638
+ """
639
+ Args:
640
+ probs: (num_query_tokens_in_batch, num_vocab)
641
+ logprobs: (num_query_tokens_in_batch, num_vocab)
642
+ sampling_metadata: The metadata for a batch for sampling.
643
+ sampling_tensors: Tensors that include sampling related metadata.
644
+
645
+ Returns:
646
+ (next_token_ids, parent_seq_ids) for each seq group in a batch.
647
+ If sampling is skipped, it returns ([], [])
648
+ sampled_token_ids_tensor: A tensor of sampled token ids.
649
+ """
650
return _sample_with_torch(
651
probs,
652
logprobs,
@@ -626,56 +683,97 @@ def _get_logprobs(
683
logprobs: torch.Tensor,
684
sampling_metadata: SamplingMetadata,
685
sample_results: List[Tuple[List[int], List[int]]],
629
-) -> Tuple[List[Optional[List[Optional[Dict[int, float]]]]], List[List[Dict[
630
- int, float]]]]:
631
- # Prepare query indices
632
- batched_logprobs_query_seq_indices: List[int] = []
633
- batched_logprobs_query_token_indices: List[int] = []
634
- # at least get one logprob for each token
686
+) -> Tuple[List[Optional[PromptLogprobs]], List[SampleLogprobs]]:
687
+ """Return sample lobprobs and prompt logprobs.
688
+
689
+ The logic consists of 3 parts.
690
+ - Select indices to compute logprob from, ranks of token ids, and
691
+ the top k token ids from logprobs.
692
+ - Compute prompt logprobs if required.
693
+ - Compute sample logprobs if required.
694
+
695
+ Args:
696
+ logprobs: (num_query_tokens_across_batch, num_vocab). Each query token's
697
+ logprob per vocab. Sequence groups' query tokens are batched in a
698
+ single flattened tensor. For example, assuming there are N
699
+ seq groups, it is sorted by prefill tokens for seq_group_1 (if
700
+ prompt logprob is enabled), decode tokens for seq_group_1 (if
701
+ sampling is required), prefill tokens for seq_group_2, ...
702
+ sampling_metadata: The sampling metadata.
703
+ sample_results: (num_seq_groups) The tuple of (next_token_ids,
704
+ parent_ids) for each sequence group. When beam search is enabled,
705
+ sample_results can contain different number of seq_ids from
706
+ sampling_metadata.seq_groups. It is because beam search creates
707
+ 2 * BEAM_WIDTH number of samples (whereas there are only up to
708
+ BEAM_WIDTH number of seq_ids).
709
+
710
+ Returns:
711
+ A tuple of prompt and sample logprobs per sequence group in a batch.
712
+ """
713
+ # The index of query token to calculate logprobs. It includes both
714
+ # prompt and sample logprob indices.
715
+ query_indices: List[int] = []
716
+ # The next token ids to get the logprob value from.
717
+ next_token_ids: List[int] = []
718
+ # The largest requested number of logprobs. We find logprobs as many as the
719
+ # largest num logprobs in this API.
720
largest_num_logprobs = 1
636
- sample_idx = 0
637
- for i, (seq_group, sample_result) in enumerate(
638
- zip(sampling_metadata.seq_groups, sample_results)):
639
- seq_ids, sampling_params = seq_group
640
- next_token_ids, parent_ids = sample_result
641
- num_parent_seqs = len(seq_ids)
642
- if (i < sampling_metadata.num_prompts
721
+
722
+ # Select indices to compute logprob from, ranks of token ids, and the top
723
+ # k token ids from logprobs.
724
+ for (seq_group, sample_result) in zip(sampling_metadata.seq_groups,
725
+ sample_results):
726
+ sampling_params = seq_group.sampling_params
727
+
728
+ # Update indices and tokens for prompt logprobs.
729
+ if (seq_group.is_prompt
730
and sampling_params.prompt_logprobs is not None):
731
largest_num_logprobs = max(largest_num_logprobs,
732
sampling_params.prompt_logprobs)
646
- prompt_len = sampling_metadata.prompt_lens[i]
647
- prompt_tokens = sampling_metadata.seq_data[
648
- seq_ids[0]].prompt_token_ids
649
- batched_logprobs_query_seq_indices.extend(
650
- sample_idx + j for j in range(prompt_len - 1))
651
- batched_logprobs_query_token_indices.extend(
652
- token_id for token_id in prompt_tokens[1:])
653
- sample_idx += prompt_len - 1
654
- batched_logprobs_query_seq_indices.extend(
655
- [sample_idx + parent_id for parent_id in parent_ids])
656
- batched_logprobs_query_token_indices.extend(next_token_ids)
657
- if sampling_params.logprobs is not None:
658
- largest_num_logprobs = max(largest_num_logprobs,
659
- sampling_params.logprobs)
660
- sample_idx += num_parent_seqs
661
- assert sample_idx == logprobs.size(0)
662
-
663
- batched_logprobs_query_seq_indices_gpu = torch.tensor(
664
- batched_logprobs_query_seq_indices, device=logprobs.device)
665
- batched_logprobs_query_token_indices_gpu = torch.tensor(
666
- batched_logprobs_query_token_indices, device=logprobs.device)
667
-
668
- # Batched query for logprobs of selected token
669
- batched_logprobs_query_result = logprobs[[
670
- batched_logprobs_query_seq_indices_gpu,
671
- batched_logprobs_query_token_indices_gpu
733
+ next_prompt_tokens = _get_next_prompt_tokens(seq_group)
734
+ query_indices.extend(seq_group.prompt_logprob_indices)
735
+ next_token_ids.extend(next_prompt_tokens)
736
+
737
+ # Update indices and next tokenes for sample logprob.
738
+ if seq_group.do_sample:
739
+ token_ids, parent_seq_ids = sample_result
740
+ # NOTE: We cannot directly use sample_indices because
741
+ # sample_indices only contain parent seq_ids of a previous step.
742
+ # The current step may have different number of seq_ids, and
743
+ # we can obtain it from `sample_result[1]`.
744
+ query_idx = seq_group.sample_indices[0]
745
+ query_indices.extend(
746
+ [query_idx + parent_id for parent_id in parent_seq_ids])
747
+ next_token_ids.extend(token_ids)
748
+
749
+ if sampling_params.logprobs is not None:
750
+ largest_num_logprobs = max(largest_num_logprobs,
751
+ sampling_params.logprobs)
752
+
753
+ assert len(next_token_ids) == len(query_indices)
754
+
755
+ if len(query_indices) == 0:
756
+ empty_sampled_logprob = []
757
+ empty_prompt_logprob = None
758
+ return [empty_prompt_logprob], [empty_sampled_logprob]
759
+
760
+ query_indices_gpu = torch.tensor(query_indices, device=logprobs.device)
761
+ next_token_ids_gpu = torch.tensor(next_token_ids, device=logprobs.device)
762
+
763
+ # (num_selected_query_tokens, num_logprobs). Note that query_indices can
764
+ # contain duplicates if beam search is enabled.
765
+ selected_logprobs = logprobs[[
766
+ query_indices_gpu,
767
+ next_token_ids_gpu,
768
]]
769
+ ranks = _get_ranks(
770
+ logprobs[query_indices_gpu],
771
+ next_token_ids_gpu,
772
+ )
773
+ assert selected_logprobs.shape[0] == ranks.shape[0]
774
674
- batched_ranks_query_result = _get_ranks(
675
- logprobs[batched_logprobs_query_seq_indices_gpu],
676
- batched_logprobs_query_token_indices_gpu)
677
-
678
- # Batched query for logprobs of topk tokens
775
+ # Logprobs of topk tokens for a batch of sequence groups.
776
+ # (num_query_tokens_across_batch).
777
if largest_num_logprobs > 0:
778
top_logprobs, top_token_ids = torch.topk(logprobs,
779
largest_num_logprobs,
@@ -685,79 +783,136 @@ def _get_logprobs(
783
else:
784
top_logprobs, top_token_ids = None, None
785
688
- batched_logprobs_query_result = batched_logprobs_query_result.cpu()
689
- batched_ranks_query_result = batched_ranks_query_result.cpu()
690
-
691
- # Gather results
692
- result_prompt_logprobs: List[Optional[PromptLogprobs]] = []
693
- result_sample_logprobs: List[SampleLogprobs] = []
694
- sample_idx = 0
695
- query_result_idx = 0
696
- for i, (seq_group, sample_result) in enumerate(
697
- zip(sampling_metadata.seq_groups, sample_results)):
698
- seq_ids, sampling_params = seq_group
699
- next_token_ids, parent_ids = sample_result
786
+ selected_logprobs = selected_logprobs.cpu()
787
+ ranks = ranks.cpu()
788
+
789
+ # Find prompt/sample logprobs.
790
+ prompt_logprobs_per_seq_group: List[Optional[PromptLogprobs]] = []
791
+ sample_logprobs_per_seq_group: List[SampleLogprobs] = []
792
+ top_logprob_idx = 0
793
+ selected_logprobs_idx = 0
794
+
795
+ for seq_group, sample_result in zip(sampling_metadata.seq_groups,
796
+ sample_results):
797
+ (prompt_logprobs, top_logprob_idx,
798
+ selected_logprobs_idx) = _get_prompt_logprob_if_needed(
799
+ seq_group, selected_logprobs, ranks, top_token_ids, top_logprobs,
800
+ selected_logprobs_idx, top_logprob_idx)
801
+ prompt_logprobs_per_seq_group.append(prompt_logprobs)
802
+
803
+ (sampled_logprobs, top_logprob_idx,
804
+ selected_logprobs_idx) = _get_sampled_logprob_if_needed(
805
+ seq_group, sample_result, selected_logprobs, ranks, top_token_ids,
806
+ top_logprobs, selected_logprobs_idx, top_logprob_idx)
807
+ sample_logprobs_per_seq_group.append(sampled_logprobs)
808
+
809
+ return prompt_logprobs_per_seq_group, sample_logprobs_per_seq_group
810
+
811
+
812
+def _get_prompt_logprob_if_needed(
813
+ seq_group: SequenceGroupToSample,
814
+ selected_logprobs: torch.Tensor,
815
+ ranks: torch.Tensor,
816
+ top_token_ids: torch.Tensor,
817
+ top_logprobs: torch.Tensor,
818
+ selected_logprobs_idx: int,
819
+ top_logprob_idx: int,
820
+):
821
+ """Compute the prompt logprob from a sequence group if needed."""
822
+ sampling_params = seq_group.sampling_params
823
+ is_prompt = seq_group.is_prompt
824
+
825
+ # Find prompt logprobs
826
+ prompt_logprobs: Optional[PromptLogprobs] = None
827
+ if (is_prompt and sampling_params.prompt_logprobs is not None):
828
+ prompt_logprobs = []
829
+ num_logprobs = sampling_params.prompt_logprobs
830
+ next_prompt_tokens = _get_next_prompt_tokens(seq_group)
831
+ for token_id in next_prompt_tokens:
832
+ # Calculate the prompt logprob of the real prompt tokens.
833
+ # Use tuple here for performance (to use to_list()).
834
+ # {token_id: (logprob, rank_from_vocab)}
835
+ prompt_logprobs_dict: Dict[int, Tuple[float, int]] = {
836
+ token_id: (selected_logprobs[selected_logprobs_idx].item(),
837
+ ranks[selected_logprobs_idx].item())
838
+ }
839
701
- # Prompt logprobs
702
- if (i < sampling_metadata.num_prompts
703
- and sampling_params.prompt_logprobs is not None):
704
- num_logprobs = sampling_params.prompt_logprobs
705
- prompt_tokens = sampling_metadata.seq_data[
706
- seq_ids[0]].prompt_token_ids
707
- group_prompt_logprobs: PromptLogprobs = [None]
708
- for token_id in prompt_tokens[1:]:
709
- prompt_logprobs_dict = {
710
- token_id:
711
- (batched_logprobs_query_result[query_result_idx].item(),
712
- batched_ranks_query_result[query_result_idx].item())
713
- }
714
- if num_logprobs > 0:
715
- prompt_logprobs_dict.update(
840
+ # Add top K prompt logprobs along with its rank.
841
+ if num_logprobs > 0:
842
+ prompt_logprobs_dict.update(
843
+ zip(
844
+ top_token_ids[top_logprob_idx, :num_logprobs].tolist(),
845
zip(
717
- top_token_ids[sample_idx, :num_logprobs].tolist(),
718
- zip(
719
- top_logprobs[
720
- sample_idx, :num_logprobs].tolist(),
721
- range(1, num_logprobs + 1))))
722
- group_prompt_logprobs.append({
723
- token_id: Logprob(*logprob_rank)
724
- for token_id, logprob_rank in prompt_logprobs_dict.items()
725
- })
726
- sample_idx += 1
727
- query_result_idx += 1
728
- result_prompt_logprobs.append(group_prompt_logprobs)
729
- else:
730
- result_prompt_logprobs.append(None)
731
-
732
- # Sample logprobs
733
- num_logprobs = sampling_params.logprobs
734
- if num_logprobs is None:
735
- num_logprobs = 0
736
- group_sample_logprobs: SampleLogprobs = []
737
- for next_token_id, parent_id in zip(next_token_ids, parent_ids):
738
- sample_logprobs_dict = {
846
+ top_logprobs[
847
+ top_logprob_idx, :num_logprobs].tolist(),
848
+ # This is ranks. Since top_logprob is sorted,
849
+ # we can just use a range here.
850
+ range(1, num_logprobs + 1))))
851
+ prompt_logprobs.append({
852
+ token_id: Logprob(*logprob_and_rank)
853
+ for token_id, logprob_and_rank in prompt_logprobs_dict.items()
854
+ })
855
+ # + 1 to go to the next prompt token.
856
+ top_logprob_idx += 1
857
+ selected_logprobs_idx += 1
858
+ return prompt_logprobs, top_logprob_idx, selected_logprobs_idx
859
+
860
+
861
+def _get_sampled_logprob_if_needed(
862
+ seq_group: SequenceGroupToSample,
863
+ sample_result: Tuple[List[int], List[int]],
864
+ selected_logprobs: torch.Tensor,
865
+ ranks: torch.Tensor,
866
+ top_token_ids: torch.Tensor,
867
+ top_logprobs: torch.Tensor,
868
+ selected_logprobs_idx: int,
869
+ top_logprob_idx: int,
870
+):
871
+ """Compute the sample logprob if needed."""
872
+ seq_ids = seq_group.seq_ids
873
+ num_logprobs = seq_group.sampling_params.logprobs
874
+ if num_logprobs is None:
875
+ num_logprobs = 0
876
+ sampled_logprobs: SampleLogprobs = []
877
+ next_token_ids, parent_seq_ids = sample_result
878
+
879
+ if seq_group.do_sample:
880
+ assert len(next_token_ids) > 0
881
+ for (next_token_id, parent_id) in zip(next_token_ids, parent_seq_ids):
882
+ # Calculate the sample logprob of the real sampled tokens.
883
+ # Use tuple here for performance (to use to_list()).
884
+ # token_id: (logprob, rank_from_vocab)
885
+ sampled_logprobs_dict: Dict[int, Tuple[float, int]] = {
886
next_token_id:
740
- (batched_logprobs_query_result[query_result_idx].item(),
741
- batched_ranks_query_result[query_result_idx].item())
887
+ (selected_logprobs[selected_logprobs_idx].item(),
888
+ ranks[selected_logprobs_idx].item())
889
}
743
- query_result_idx += 1
890
+ # +1 to go to the next sampled token. Note that
891
+ # selected_logprobs can contain duplicates unlike top_logprobs
892
+ # when beam search is enabled.
893
+ selected_logprobs_idx += 1
894
+
895
+ # Second, add top K logprobs along with its rank.
896
if num_logprobs >= 0:
745
- sample_logprobs_dict.update(
897
+ sampled_logprobs_dict.update(
898
zip(
747
- top_token_ids[sample_idx +
899
+ top_token_ids[top_logprob_idx +
900
parent_id, :num_logprobs].tolist(),
901
zip(
750
- top_logprobs[sample_idx +
902
+ top_logprobs[top_logprob_idx +
903
parent_id, :num_logprobs].tolist(),
904
+ # This is rank. Since top_logprob is sorted, we
905
+ # can just use a range here.
906
range(1, num_logprobs + 1))))
753
- group_sample_logprobs.append({
754
- token_id: Logprob(*logprob_rank)
755
- for token_id, logprob_rank in sample_logprobs_dict.items()
907
+ sampled_logprobs.append({
908
+ token_id: Logprob(*logprob_and_rank)
909
+ for token_id, logprob_and_rank in
910
+ sampled_logprobs_dict.items()
911
})
757
- result_sample_logprobs.append(group_sample_logprobs)
758
- sample_idx += len(seq_ids)
759
-
760
- return result_prompt_logprobs, result_sample_logprobs
912
+ # There are len(seq_ids) number of sampled tokens for the current
913
+ # sequence group in top_logprobs. Jump to the next seq_group.
914
+ top_logprob_idx += len(seq_ids)
915
+ return sampled_logprobs, top_logprob_idx, selected_logprobs_idx
916
917
918
def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
@@ -832,7 +987,7 @@ def _build_sampler_output(
987
group_sample_logprobs) in zip(sampling_metadata.seq_groups,
988
sample_results, prompt_logprobs,
989
sample_logprobs):
835
- seq_ids, _ = seq_group
990
+ seq_ids = seq_group.seq_ids
991
next_token_ids, parent_ids = sample_result
992
seq_outputs = []
993
for parent_id, next_token_id, logprobs in zip(parent_ids,
@@ -854,3 +1009,36 @@ def _build_sampler_output(
1009
sampled_token_probs=sampled_token_probs,
1010
sampled_token_ids=sampled_token_ids,
1011
)
1012
+
1013
+
1014
+def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[str]:
1015
+ """Get a list of next prompt tokens to compute logprob from a
1016
+ given sequence group.
1017
+
1018
+ It is used to compute prompt logprob. Imagine you have logprob for each
1019
+ query token. Query token needs to know the next prompt token id to compute
1020
+ prompt logprob. This is a helper to obtain next prompt token ids.
1021
+
1022
+ This API has to be used only when the caller knows seq_group is in prefill
1023
+ stage.
1024
+
1025
+ Returns:
1026
+ A list of next prompt tokens to compute logprob.
1027
+ """
1028
+ assert seq_group.is_prompt, (
1029
+ "Caller should ensure the sequence group is in a prefill stage.")
1030
+ seq_ids = seq_group.seq_ids
1031
+ subquery_len = seq_group.subquery_len
1032
+ assert subquery_len is not None
1033
+ # prompt has only 1 seq id.
1034
+ assert len(seq_ids) == 1
1035
+ seq_data = seq_group.seq_data[seq_ids[0]]
1036
+ computed_len = seq_data.get_num_computed_tokens()
1037
+ prompt_tokens = seq_data.prompt_token_ids
1038
+ # +1 because we are looking for a next prompt token.
1039
+ next_token_index_start = computed_len + 1
1040
+ next_token_index_end = min(computed_len + subquery_len + 1,
1041
+ len(prompt_tokens))
1042
+ next_prompt_tokens = prompt_tokens[
1043
+ next_token_index_start:next_token_index_end]
1044
+ return next_prompt_tokens
vllm/model_executor/sampling_metadata.py
+284
-65
@@ -6,57 +6,275 @@ import torch
6
7
from vllm.model_executor.layers.ops.sample import get_num_triton_sampler_splits
8
from vllm.sampling_params import SamplingParams, SamplingType
9
-from vllm.sequence import SequenceData
10
-from vllm.utils import is_pin_memory_available
9
+from vllm.sequence import SequenceData, SequenceGroupMetadata
10
+from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
11
+ maybe_expand_dim)
12
13
_SAMPLING_EPS = 1e-5
14
_SEED_0_REPLACEMENT = 3403598558
15
16
17
+@dataclass
18
+class SequenceGroupToSample:
19
+ # Sequence ids for the sequence group in a previous step.
20
+ seq_ids: List[int]
21
+ sampling_params: SamplingParams
22
+ # seq_id -> sequence data.
23
+ seq_data: Dict[int, SequenceData]
24
+ # The length of the prompt of the sequence group. None if it is in a decode
25
+ # stage.
26
+ prompt_len: Optional[int]
27
+ # The length of the query tokens to compute in the current step. None if it
28
+ # is in a decode stage. The length of subquery_len <= prompt_len.
29
+ subquery_len: Optional[int]
30
+ # A random number generator for sampling.
31
+ generator: Optional[torch.Generator]
32
+ # True if the sequence group is in prefill stage. False if it is in a
33
+ # decode stage.
34
+ is_prompt: bool
35
+ # Query token indices from logits. to compute prompt logprob. Empty if
36
+ # prompt logprob is not required.
37
+ prompt_logprob_indices: List[int]
38
+ # Sample token indices from logits. Empty if sampling is not required.
39
+ sample_indices: List[int]
40
+
41
+ @property
42
+ def do_sample(self):
43
+ return len(self.sample_indices) > 0
44
+
45
+ def __post_init__(self):
46
+ if len(self.prompt_logprob_indices) > 0:
47
+ assert self.sampling_params.prompt_logprobs is not None
48
+ if self.is_prompt:
49
+ assert self.prompt_len is not None
50
+ assert self.subquery_len is not None
51
+
52
+
53
class SamplingMetadata:
54
"""Metadata for input sequences. Used in sampler.
55
56
+ The usage is as follow;
57
+ ```
58
+ hidden_states = execute_model(...)
59
+ logits = hidden_states[sampling_metadata.selected_token_indices]
60
+ sample(logits)
61
+
62
+ def sample(logits):
63
+ # Use categorized_sample_indices for sampling....
64
+ ```
65
+
66
Args:
20
- seq_groups: List of (seq_ids, sampling_params).
21
- seq_data: Seq_id -> SequenceData.
22
- prompt_lens: Lengths of prompts.
23
- selected_token_indices: Token indices selected for sampling.
67
+ seq_groups: List of batched sequence groups.
68
+ selected_token_indices: (num_query_tokens_to_logprob). Indices to find
69
+ logits from the initial model output hidden states.
70
categorized_sample_indices: SamplingType -> token indices to sample.
25
- generators: List of torch.Generators to use for seeded sampling
26
- perform_sampling: Whether to perform sampling. This option is used to
27
- make the sampling only happens in the driver worker, and disable
28
- sampling in other worker processes.
71
+ Each token indices is 2D tensor of (num_indices, num_indices) where
72
+ the first item means the sample index within the returned logit
73
+ (before pruning padding), and the second item means the sample
74
+ index after pruning using selected_token_indices.
75
+ For example, if the returned logit is [1, 2, 3], and we select
76
+ [1, 2] for sampling, the pruned logit will be [2, 3]. In this case,
77
+ The first tuple is [1, 2] (sampled index within original logit),
78
+ and the second tuple is [0, 1] (sampled index within pruned logit).
79
+ num_prompts: Number of prompt sequence groups in seq_groups.
80
"""
81
82
def __init__(
83
self,
33
- seq_groups: Optional[List[Tuple[List[int], SamplingParams]]],
34
- seq_data: Optional[Dict[int, SequenceData]],
35
- prompt_lens: Optional[List[int]],
84
+ seq_groups: List[SequenceGroupToSample],
85
selected_token_indices: torch.Tensor,
37
- categorized_sample_indices: Optional[Dict[SamplingType, torch.Tensor]],
38
- generators: Optional[List[torch.Generator]] = None,
39
- perform_sampling: bool = True,
86
+ categorized_sample_indices: Dict[SamplingType, torch.Tensor],
87
+ num_prompts: int,
88
) -> None:
89
self.seq_groups = seq_groups
42
- self.seq_data = seq_data
43
- self.prompt_lens = prompt_lens
90
self.selected_token_indices = selected_token_indices
91
self.categorized_sample_indices = categorized_sample_indices
46
- self.generators = generators
47
- self.perform_sampling = perform_sampling
92
+ self.num_prompts = num_prompts
93
49
- self.num_prompts = len(prompt_lens) if prompt_lens is not None else 0
94
+ @staticmethod
95
+ def prepare(
96
+ seq_group_metadata_list: List[SequenceGroupMetadata],
97
+ prompt_lens: List[int],
98
+ subquery_lens: Optional[List[int]],
99
+ device: str,
100
+ pin_memory: bool,
101
+ ) -> "SamplingMetadata":
102
+ (
103
+ seq_groups,
104
+ selected_token_indices,
105
+ categorized_sample_indices,
106
+ num_prompts,
107
+ ) = _prepare_seq_groups(seq_group_metadata_list, prompt_lens,
108
+ subquery_lens, device)
109
+ selected_token_indices = async_tensor_h2d(selected_token_indices,
110
+ dtype=torch.long,
111
+ target_device=device,
112
+ pin_memory=pin_memory)
113
+ categorized_sample_indices = {
114
+ t: maybe_expand_dim(
115
+ async_tensor_h2d(seq_ids,
116
+ dtype=torch.int,
117
+ target_device=device,
118
+ pin_memory=pin_memory), 2, 2)
119
+ for t, seq_ids in categorized_sample_indices.items()
120
+ }
121
+
122
+ sampling_metadata = SamplingMetadata(
123
+ seq_groups=seq_groups,
124
+ selected_token_indices=selected_token_indices,
125
+ categorized_sample_indices=categorized_sample_indices,
126
+ num_prompts=num_prompts,
127
+ )
128
+ return sampling_metadata
129
130
def __repr__(self) -> str:
131
return (
132
"SamplingMetadata("
133
f"seq_groups={self.seq_groups}, "
55
- f"seq_data={self.seq_data}, "
56
- f"prompt_lens={self.prompt_lens}, "
134
f"selected_token_indices={self.selected_token_indices}, "
58
- f"categorized_sample_indices={self.categorized_sample_indices}), "
59
- f"perform_sampling={self.perform_sampling})")
135
+ f"categorized_sample_indices={self.categorized_sample_indices}), ")
136
+
137
+
138
+def _prepare_seq_groups(
139
+ seq_group_metadata_list: List[SequenceGroupMetadata],
140
+ prompt_lens: List[int],
141
+ subquery_lens: Optional[List[int]],
142
+ device: str,
143
+) -> Tuple[List[SequenceGroupToSample], List[int], Dict[
144
+ SamplingType, List[Tuple[int, int]]], int]:
145
+ """Prepare sequence groups and indices for sampling.
146
+
147
+ Args:
148
+ seq_group_metadata_list: A list of sequence group to batch.
149
+ prompt_lens: A list of prompt lens per sequence group.
150
+ Index of prompt len should match with seq_group_metadata_list.
151
+ subquery_lens: A list of query lengths. Prompt lens include the length
152
+ of entire prompt tokens, and it could be shorter.
153
+ device: A device to use for random number generator,
154
+ `SequenceGroupToSample.generator`.
155
+
156
+ Returns:
157
+ seq_groups: A list of sequence group to sample.
158
+ selected_token_indices: See the definition from `SamplingMetadata`.
159
+ categorized_sample_indices: See the definition from `SamplingMetadata`.
160
+ num_prompts: Total number of prompts from `seq_group_metadata_list`.
161
+ """
162
+ # Batched sequence groups for the current model forward stsep.
163
+ seq_groups: List[SequenceGroupToSample] = []
164
+ # A list of token indices to sample/compute logprob. It is used to
165
+ # prune the outcome logits from the model for the performance.
166
+ selected_token_indices: List[int] = []
167
+ # Used for selected_token_indices.
168
+ model_output_idx = 0
169
+
170
+ # Sampling type -> (
171
+ # indices to sample/prompt logprob within pruned output logits,
172
+ # indices to sample within pruned logits)
173
+ categorized_sample_indices: Dict[SamplingType, List[Tuple[int, int]]] = {
174
+ t: []
175
+ for t in SamplingType
176
+ }
177
+ # Index of logits to compute logprob. Logits include both prompt logprob
178
+ # and sample logprob indices.
179
+ logit_idx = 0
180
+ # Index to sample from a sample tensor. It is used by triton sample kernel.
181
+ # See `_sample_with_triton_kernel` for more details.
182
+ sample_idx = 0
183
+ # Total number of prompts from given sequence groups.
184
+ num_prompts = 0
185
+
186
+ for i, seq_group_metadata in enumerate(seq_group_metadata_list):
187
+ seq_ids = list(seq_group_metadata.seq_data.keys())
188
+ sampling_params = seq_group_metadata.sampling_params
189
+ is_prompt = seq_group_metadata.is_prompt
190
+ generator: Optional[torch.Generator] = None
191
+ # If the current seq group is in decode stage, it is None.
192
+ prompt_len: Optional[int] = None
193
+ subquery_len: Optional[int] = None
194
+ prompt_logprob_indices: List[int] = []
195
+ sample_indices: List[int] = []
196
+ do_sample = seq_group_metadata.do_sample
197
+
198
+ if seq_group_metadata.is_prompt:
199
+ if sampling_params.seed is not None:
200
+ seq_group_metadata.state.generator = torch.Generator(
201
+ device=device).manual_seed(sampling_params.seed)
202
+
203
+ num_prompts += 1
204
+ num_prefill_sample = len(seq_ids)
205
+ assert num_prefill_sample == 1
206
+ assert subquery_lens is not None and prompt_lens is not None
207
+ subquery_len, prompt_len = subquery_lens[i], prompt_lens[i]
208
+ # If we need sampling, exclude num_prefill_sample tokens from
209
+ # prompt logprob.
210
+ prompt_logprob_len = (subquery_len - num_prefill_sample
211
+ if do_sample else subquery_len)
212
+ sample_len = num_prefill_sample if do_sample else 0
213
+ else:
214
+ # Decode
215
+ prompt_logprob_len = 0
216
+ sample_len = len(seq_ids) if do_sample else 0
217
+
218
+ # Update indices to select from the model output.
219
+ """
220
+ This blocks computes selected_token_indices which is used in the
221
+ following way.
222
+
223
+ hidden_states = model(...)
224
+ logits = hidden_states[selected_token_indices]
225
+ """
226
+
227
+ if sampling_params.prompt_logprobs:
228
+ selected_token_indices.extend(
229
+ range(model_output_idx, model_output_idx + prompt_logprob_len))
230
+ model_output_idx += prompt_logprob_len
231
+ if do_sample:
232
+ selected_token_indices.extend(
233
+ range(model_output_idx, model_output_idx + sample_len))
234
+ model_output_idx += sample_len
235
+
236
+ # We now find indices for logprob computation and sampling.
237
+ """
238
+ This block computes categorized_sample_indices which is used in the
239
+ following way.
240
+
241
+ hidden_states = model(...)
242
+ logits = hidden_states[selected_token_indices]
243
+ def sample(logits):
244
+ # Use categorized_sample_indices for sampling.
245
+ # prompt_logprob_indices to find prompt logprob indices.
246
+ # sample_indices to find sample indices.
247
+ """
248
+
249
+ if sampling_params.prompt_logprobs is not None:
250
+ prompt_logprob_indices.extend(
251
+ range(logit_idx, logit_idx + prompt_logprob_len))
252
+ logit_idx += prompt_logprob_len
253
+ if do_sample:
254
+ sample_indices.extend(range(logit_idx, logit_idx + sample_len))
255
+ categorized_sample_indices[sampling_params.sampling_type].extend(
256
+ list(
257
+ zip(range(logit_idx, logit_idx + sample_len),
258
+ range(sample_idx, sample_idx + sample_len))))
259
+ logit_idx += sample_len
260
+ sample_idx += sample_len
261
+
262
+ if sampling_params.seed is not None:
263
+ generator = seq_group_metadata.state.generator
264
+
265
+ seq_groups.append(
266
+ SequenceGroupToSample(
267
+ seq_ids=seq_ids,
268
+ sampling_params=sampling_params,
269
+ seq_data=seq_group_metadata.seq_data,
270
+ prompt_len=prompt_len,
271
+ subquery_len=subquery_len,
272
+ generator=generator,
273
+ is_prompt=is_prompt,
274
+ prompt_logprob_indices=list(prompt_logprob_indices),
275
+ sample_indices=list(sample_indices)))
276
+ return (seq_groups, selected_token_indices, categorized_sample_indices,
277
+ num_prompts)
278
279
280
@dataclass
@@ -112,11 +330,10 @@ class SamplingTensors:
330
seeds_to_generate = (extra_seeds_to_generate +
331
get_num_triton_sampler_splits(vocab_size))
332
115
- sample_indices_start_idx = 0
333
assert sampling_metadata.seq_groups is not None
117
- assert sampling_metadata.seq_data is not None
118
- for i, seq_group in enumerate(sampling_metadata.seq_groups):
119
- seq_ids, sampling_params = seq_group
334
+ for seq_group in sampling_metadata.seq_groups:
335
+ seq_ids = seq_group.seq_ids
336
+ sampling_params = seq_group.sampling_params
337
temperature = sampling_params.temperature
338
p = sampling_params.presence_penalty
339
f = sampling_params.frequency_penalty
@@ -145,45 +362,46 @@ class SamplingTensors:
362
or abs(r - 1.0) >= _SAMPLING_EPS):
363
do_penalties = True
364
148
- if (i < sampling_metadata.num_prompts
365
+ is_prompt = seq_group.is_prompt
366
+ if (seq_group.is_prompt
367
and sampling_params.prompt_logprobs is not None):
368
# For tokens in the prompt that we only need to get
369
# their logprobs
152
- assert sampling_metadata.prompt_lens is not None
153
- prompt_len = sampling_metadata.prompt_lens[i]
154
- temperatures += [temperature] * (prompt_len - 1)
155
- top_ps += [top_p] * (prompt_len - 1)
156
- top_ks += [top_k] * (prompt_len - 1)
157
- min_ps += [min_p] * (prompt_len - 1)
158
- presence_penalties += [0] * (prompt_len - 1)
159
- frequency_penalties += [0] * (prompt_len - 1)
160
- repetition_penalties += [1] * (prompt_len - 1)
161
- prompt_tokens.extend([] for _ in range(prompt_len - 1))
162
- output_tokens.extend([] for _ in range(prompt_len - 1))
163
- for seq_id in seq_ids:
164
- seq_data = sampling_metadata.seq_data[seq_id]
165
- prompt_tokens.append(seq_data.prompt_token_ids)
166
- output_tokens.append(seq_data.output_token_ids)
167
- temperatures += [temperature] * len(seq_ids)
168
- top_ps += [top_p] * len(seq_ids)
169
- top_ks += [top_k] * len(seq_ids)
170
- min_ps += [min_p] * len(seq_ids)
171
- presence_penalties += [p] * len(seq_ids)
172
- frequency_penalties += [f] * len(seq_ids)
173
- repetition_penalties += [r] * len(seq_ids)
174
-
175
- is_prompt = i < sampling_metadata.num_prompts
370
+ subquery_len = seq_group.subquery_len
371
+ assert subquery_len is not None
372
+ prefill_len = len(seq_group.prompt_logprob_indices)
373
+ temperatures += [temperature] * prefill_len
374
+ top_ps += [top_p] * prefill_len
375
+ top_ks += [top_k] * prefill_len
376
+ min_ps += [min_p] * prefill_len
377
+ presence_penalties += [0] * prefill_len
378
+ frequency_penalties += [0] * prefill_len
379
+ repetition_penalties += [1] * prefill_len
380
+ prompt_tokens.extend([] for _ in range(prefill_len))
381
+ output_tokens.extend([] for _ in range(prefill_len))
382
+
383
+ if seq_group.do_sample:
384
+ sample_lens = len(seq_group.sample_indices)
385
+ assert sample_lens == len(seq_ids)
386
+ for seq_id in seq_ids:
387
+ seq_data = seq_group.seq_data[seq_id]
388
+ prompt_tokens.append(seq_data.prompt_token_ids)
389
+ output_tokens.append(seq_data.output_token_ids)
390
+ temperatures += [temperature] * len(seq_ids)
391
+ top_ps += [top_p] * len(seq_ids)
392
+ top_ks += [top_k] * len(seq_ids)
393
+ min_ps += [min_p] * len(seq_ids)
394
+ presence_penalties += [p] * len(seq_ids)
395
+ frequency_penalties += [f] * len(seq_ids)
396
+ repetition_penalties += [r] * len(seq_ids)
397
+
398
if is_prompt:
399
prompt_best_of.append(sampling_params.best_of)
178
- assert sampling_metadata.prompt_lens is not None
179
- prompt_len = sampling_metadata.prompt_lens[i]
400
+ subquery_len = seq_group.subquery_len
401
+ assert subquery_len is not None
402
181
- if sampling_params.prompt_logprobs is not None:
182
- # NOTE: the sampling position is the last token
183
- # in the prompt
184
- sample_indices_start_idx += prompt_len - 1
403
for seq_id in seq_ids:
186
- seq_data = sampling_metadata.seq_data[seq_id]
404
+ seq_data = seq_group.seq_data[seq_id]
405
extra_entropy = extra_entropy or ()
406
seq_seeds = cls._get_sequence_seeds(
407
seed,
@@ -193,8 +411,7 @@ class SamplingTensors:
411
seeds_to_generate=seeds_to_generate,
412
is_greedy=is_greedy)
413
sampling_seeds.append(seq_seeds)
196
- sample_indices.append(sample_indices_start_idx)
197
- sample_indices_start_idx += 1
414
+ sample_indices.extend(seq_group.sample_indices)
415
416
sampling_tensors = SamplingTensors.from_lists(
417
temperatures, top_ps, top_ks, min_ps, presence_penalties,
@@ -217,12 +434,14 @@ class SamplingTensors:
434
# Note that the performance will be very bad without
435
# pinned memory.
436
pin_memory = is_pin_memory_available()
220
- prompt_max_len = max(len(tokens) for tokens in prompt_tokens)
437
+ prompt_max_len = max([len(tokens) for tokens in prompt_tokens],
438
+ default=0)
439
prompt_padded_tokens = [
440
tokens + [vocab_size] * (prompt_max_len - len(tokens))
441
for tokens in prompt_tokens
442
]
225
- output_max_len = max(len(tokens) for tokens in output_tokens)
443
+ output_max_len = max([len(tokens) for tokens in output_tokens],
444
+ default=0)
445
output_padded_tokens = [
446
tokens + [vocab_size] * (output_max_len - len(tokens))
447
for tokens in output_tokens
vllm/sequence.py
+10
-1
@@ -28,7 +28,10 @@ class Logprob:
28
decoded_token: Optional[str] = None
29
30
31
+# {token_id -> logprob} per each sequence group. None if the corresponding
32
+# sequence group doesn't require prompt logprob.
33
PromptLogprobs = List[Optional[Dict[int, Logprob]]]
34
+# {token_id -> logprob} for each sequence group.
35
SampleLogprobs = List[Dict[int, Logprob]]
36
37
@@ -215,7 +218,7 @@ class Sequence:
218
self.eos_token_id = eos_token_id
219
self.lora_request = lora_request
220
218
- self.data = SequenceData(prompt_token_ids)
221
+ self.data: SequenceData = SequenceData(prompt_token_ids)
222
self.output_logprobs: SampleLogprobs = []
223
self.output_text = ""
224
@@ -559,6 +562,9 @@ class SequenceGroupMetadata:
562
sampling_params: The sampling parameters used to generate the outputs.
563
block_tables: The block tables. (Seq id -> list of physical block
564
numbers)
565
+ do_sample: True if sampling is required. Sampling is not required when
566
+ e.g., prefill is chunked, and the current iteration only computes
567
+ query tokens for prefill, we don't need sampling.
568
token_chunk_size: The number of tokens to be processed (per sequence).
569
None if chunking is not required.
570
state: Internal state tied to this sequence group.
@@ -573,6 +579,7 @@ class SequenceGroupMetadata:
579
seq_data: Dict[int, SequenceData],
580
sampling_params: SamplingParams,
581
block_tables: Dict[int, List[int]],
582
+ do_sample: bool = True,
583
token_chunk_size: Optional[int] = None,
584
lora_request: Optional[LoRARequest] = None,
585
computed_block_nums: Optional[List[int]] = None,
@@ -589,6 +596,7 @@ class SequenceGroupMetadata:
596
self.multi_modal_data = multi_modal_data
597
self.state = SequenceGroupState() if state is None else state
598
self._token_chunk_size = token_chunk_size
599
+ self.do_sample = do_sample
600
601
if self._token_chunk_size is None:
602
if is_prompt:
@@ -650,6 +658,7 @@ class SequenceGroupOutput:
658
prompt_logprobs: Optional[PromptLogprobs],
659
) -> None:
660
self.samples = samples
661
+ # Prompt logprob for each prompt query token.
662
self.prompt_logprobs = prompt_logprobs
663
664
def __repr__(self) -> str:
vllm/worker/cpu_model_runner.py
+15
-101
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Optional, Tuple
1
+from typing import List, Optional, Tuple
2
3
import torch
4
from torch import nn
@@ -10,9 +10,8 @@ from vllm.distributed import broadcast_tensor_dict
10
from vllm.logger import init_logger
11
from vllm.model_executor import SamplingMetadata
12
from vllm.model_executor.model_loader import get_model
13
-from vllm.sampling_params import SamplingParams, SamplingType
14
-from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15
-from vllm.utils import make_tensor_with_pad, maybe_expand_dim
13
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
14
+from vllm.utils import make_tensor_with_pad
15
16
logger = init_logger(__name__)
17
@@ -38,6 +37,8 @@ class CPUModelRunner:
37
self.model_config = model_config
38
self.parallel_config = parallel_config
39
self.scheduler_config = scheduler_config
40
+ # Currently, CPU worker doesn't support chunked prefill.
41
+ assert self.scheduler_config.chunked_prefill_enabled is False
42
self.lora_config = lora_config
43
self.vision_language_config = vision_language_config
44
self.load_config = load_config
@@ -252,99 +253,6 @@ class CPUModelRunner:
253
attn_metadata,
254
)
255
255
- def _prepare_sample(
256
- self,
257
- seq_group_metadata_list: List[SequenceGroupMetadata],
258
- prompt_lens: List[int],
259
- ) -> SamplingMetadata:
260
- seq_groups: List[Tuple[List[int], SamplingParams]] = []
261
- selected_token_indices: List[int] = []
262
- generators: List[torch.Generator] = []
263
- selected_token_start_idx = 0
264
- categorized_sample_indices: Dict[SamplingType,
265
- List[Tuple[int, int]]] = {
266
- t: []
267
- for t in SamplingType
268
- }
269
- categorized_sample_indices_start_idx = 0
270
- categorized_sampled_token_indices_start_idx = 0
271
-
272
- for i, seq_group_metadata in enumerate(seq_group_metadata_list):
273
- seq_ids = list(seq_group_metadata.seq_data.keys())
274
- sampling_params = seq_group_metadata.sampling_params
275
- seq_groups.append((seq_ids, sampling_params))
276
-
277
- if seq_group_metadata.is_prompt:
278
- assert len(seq_ids) == 1
279
- subquery_len = prompt_lens[i]
280
- if sampling_params.prompt_logprobs is not None:
281
- # NOTE: prompt token positions do not need sample, skip
282
- categorized_sample_indices_start_idx += subquery_len - 1
283
-
284
- categorized_sample_indices[
285
- sampling_params.sampling_type].append(
286
- (categorized_sample_indices_start_idx,
287
- categorized_sampled_token_indices_start_idx))
288
- categorized_sample_indices_start_idx += 1
289
- categorized_sampled_token_indices_start_idx += 1
290
-
291
- if sampling_params.prompt_logprobs is not None:
292
- selected_token_indices.extend(
293
- range(selected_token_start_idx,
294
- selected_token_start_idx + subquery_len - 1))
295
- selected_token_indices.append(selected_token_start_idx +
296
- subquery_len - 1)
297
- selected_token_start_idx += subquery_len
298
-
299
- if sampling_params.seed is not None:
300
- seq_group_metadata.state.generator = torch.Generator(
301
- device=self.device).manual_seed(sampling_params.seed)
302
- else:
303
- num_seqs = len(seq_ids)
304
- selected_token_indices.extend(
305
- range(selected_token_start_idx,
306
- selected_token_start_idx + num_seqs))
307
- selected_token_start_idx += num_seqs
308
-
309
- categorized_sample_indices[
310
- sampling_params.sampling_type].extend(
311
- zip(
312
- range(
313
- categorized_sample_indices_start_idx,
314
- categorized_sample_indices_start_idx +
315
- num_seqs),
316
- range(
317
- categorized_sampled_token_indices_start_idx,
318
- categorized_sampled_token_indices_start_idx +
319
- num_seqs)))
320
- categorized_sample_indices_start_idx += num_seqs
321
- categorized_sampled_token_indices_start_idx += num_seqs
322
-
323
- if sampling_params.seed is not None:
324
- generators.append(seq_group_metadata.state.generator)
325
-
326
- selected_token_indices = torch.tensor(selected_token_indices,
327
- dtype=torch.long)
328
-
329
- categorized_sample_indices = {
330
- t: maybe_expand_dim(torch.tensor(seq_ids, dtype=torch.int), 2, 2)
331
- for t, seq_ids in categorized_sample_indices.items()
332
- }
333
-
334
- seq_data: Dict[int, SequenceData] = {}
335
- for seq_group_metadata in seq_group_metadata_list:
336
- seq_data.update(seq_group_metadata.seq_data)
337
-
338
- sampling_metadata = SamplingMetadata(
339
- seq_groups=seq_groups,
340
- seq_data=seq_data,
341
- prompt_lens=prompt_lens,
342
- selected_token_indices=selected_token_indices,
343
- categorized_sample_indices=categorized_sample_indices,
344
- generators=generators,
345
- )
346
- return sampling_metadata
347
-
256
def prepare_input_tensors(
257
self,
258
seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -364,8 +272,15 @@ class CPUModelRunner:
272
(input_tokens, input_positions,
273
attn_metadata) = self._prepare_decode(seq_group_metadata_list)
274
prompt_lens = []
367
- sampling_metadata = self._prepare_sample(seq_group_metadata_list,
368
- prompt_lens)
275
+ sampling_metadata = SamplingMetadata.prepare(
276
+ seq_group_metadata_list,
277
+ prompt_lens,
278
+ # subquery_lens is not needed if chunked prefill is not
279
+ # supported. Since CPU worker doesn't support chunked prefill
280
+ # just use prompt_lens instead.
281
+ prompt_lens,
282
+ self.device,
283
+ pin_memory=False)
284
# Broadcast the metadata.
285
metadata_dict = {
286
"input_tokens": input_tokens,
@@ -389,7 +304,6 @@ class CPUModelRunner:
304
selected_token_indices=selected_token_indices,
305
categorized_sample_indices=None,
306
generators=None,
392
- perform_sampling=False,
307
)
308
309
return (input_tokens, input_positions, attn_metadata,
@@ -421,7 +335,7 @@ class CPUModelRunner:
335
logits = self.model.compute_logits(hidden_states, sampling_metadata)
336
337
# Only perform sampling in the driver worker.
424
- if not sampling_metadata.perform_sampling:
338
+ if not self.is_driver_worker:
339
return None
340
341
# Sample the next token.
vllm/worker/model_runner.py
+9
-114
@@ -20,12 +20,11 @@ from vllm.lora.request import LoRARequest
20
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
21
from vllm.model_executor import SamplingMetadata
22
from vllm.model_executor.model_loader import get_model
23
-from vllm.sampling_params import SamplingParams, SamplingType
23
+from vllm.sampling_params import SamplingParams
24
from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
25
SequenceGroupMetadata)
26
-from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d, is_hip,
27
- is_pin_memory_available, make_tensor_with_pad,
28
- maybe_expand_dim)
26
+from vllm.utils import (CudaMemoryProfiler, is_hip, is_pin_memory_available,
27
+ make_tensor_with_pad)
28
29
logger = init_logger(__name__)
30
@@ -547,108 +546,6 @@ class ModelRunner:
546
slot_mapping=slot_mapping,
547
)
548
550
- def _prepare_sample(
551
- self,
552
- seq_group_metadata_list: List[SequenceGroupMetadata],
553
- prompt_lens: List[int],
554
- subquery_lens: Optional[List[int]],
555
- ) -> SamplingMetadata:
556
- seq_groups: List[Tuple[List[int], SamplingParams]] = []
557
- selected_token_indices: List[int] = []
558
- generators: List[torch.Generator] = []
559
- selected_token_start_idx = 0
560
- categorized_sample_indices: Dict[SamplingType,
561
- List[Tuple[int, int]]] = {
562
- t: []
563
- for t in SamplingType
564
- }
565
- categorized_sample_indices_start_idx = 0
566
- categorized_sampled_token_indices_start_idx = 0
567
-
568
- for i, seq_group_metadata in enumerate(seq_group_metadata_list):
569
- seq_ids = list(seq_group_metadata.seq_data.keys())
570
- sampling_params = seq_group_metadata.sampling_params
571
- seq_groups.append((seq_ids, sampling_params))
572
-
573
- if seq_group_metadata.is_prompt:
574
- assert len(seq_ids) == 1
575
- assert subquery_lens is not None
576
- subquery_len = subquery_lens[i]
577
- if sampling_params.prompt_logprobs is not None:
578
- # NOTE: prompt token positions do not need sample, skip
579
- categorized_sample_indices_start_idx += subquery_len - 1
580
-
581
- categorized_sample_indices[
582
- sampling_params.sampling_type].append(
583
- (categorized_sample_indices_start_idx,
584
- categorized_sampled_token_indices_start_idx))
585
- categorized_sample_indices_start_idx += 1
586
- categorized_sampled_token_indices_start_idx += 1
587
-
588
- if sampling_params.prompt_logprobs is not None:
589
- selected_token_indices.extend(
590
- range(selected_token_start_idx,
591
- selected_token_start_idx + subquery_len - 1))
592
- selected_token_indices.append(selected_token_start_idx +
593
- subquery_len - 1)
594
- selected_token_start_idx += subquery_len
595
-
596
- if sampling_params.seed is not None:
597
- seq_group_metadata.state.generator = torch.Generator(
598
- device=self.device).manual_seed(sampling_params.seed)
599
- else:
600
- num_seqs = len(seq_ids)
601
- selected_token_indices.extend(
602
- range(selected_token_start_idx,
603
- selected_token_start_idx + num_seqs))
604
- selected_token_start_idx += num_seqs
605
-
606
- categorized_sample_indices[
607
- sampling_params.sampling_type].extend(
608
- list(
609
- zip(
610
- range(
611
- categorized_sample_indices_start_idx,
612
- categorized_sample_indices_start_idx +
613
- num_seqs),
614
- range(
615
- categorized_sampled_token_indices_start_idx,
616
- categorized_sampled_token_indices_start_idx
617
- + num_seqs))))
618
- categorized_sample_indices_start_idx += num_seqs
619
- categorized_sampled_token_indices_start_idx += num_seqs
620
-
621
- if sampling_params.seed is not None:
622
- generators.append(seq_group_metadata.state.generator)
623
-
624
- selected_token_indices = async_tensor_h2d(selected_token_indices,
625
- dtype=torch.long,
626
- target_device=self.device,
627
- pin_memory=self.pin_memory)
628
-
629
- categorized_sample_indices = {
630
- t: maybe_expand_dim(
631
- async_tensor_h2d(seq_ids,
632
- dtype=torch.int,
633
- target_device=self.device,
634
- pin_memory=self.pin_memory), 2, 2)
635
- for t, seq_ids in categorized_sample_indices.items()
636
- }
637
-
638
- seq_data: Dict[int, SequenceData] = {}
639
- for seq_group_metadata in seq_group_metadata_list:
640
- seq_data.update(seq_group_metadata.seq_data)
641
-
642
- sampling_metadata = SamplingMetadata(
643
- seq_groups=seq_groups,
644
- seq_data=seq_data,
645
- prompt_lens=prompt_lens,
646
- selected_token_indices=selected_token_indices,
647
- categorized_sample_indices=categorized_sample_indices,
648
- generators=generators,
649
- )
650
- return sampling_metadata
651
-
549
def prepare_input_tensors(
550
self,
551
seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -685,9 +582,9 @@ class ModelRunner:
582
decode_lora_requests,
583
decode_slot_mapping,
584
) = self._prepare_decode(decode_reqs)
688
- sampling_metadata = self._prepare_sample(seq_group_metadata_list,
689
- prompt_lens,
690
- subquery_lens)
585
+ sampling_metadata = SamplingMetadata.prepare(
586
+ seq_group_metadata_list, prompt_lens, subquery_lens,
587
+ self.device, self.pin_memory)
588
589
if not self.scheduler_config.chunked_prefill_enabled:
590
assert (len(prefill_reqs) and len(decode_reqs)) == 0
@@ -788,12 +685,9 @@ class ModelRunner:
685
**metadata_dict)
686
sampling_metadata = SamplingMetadata(
687
seq_groups=None,
791
- seq_data=None,
792
- prompt_lens=None,
688
selected_token_indices=selected_token_indices,
689
categorized_sample_indices=None,
795
- generators=None,
796
- perform_sampling=False,
690
+ num_prompts=0,
691
)
692
693
# if it is a mixed batch, decode attn_metadata is broadcasted
@@ -852,7 +746,7 @@ class ModelRunner:
746
logits = self.model.compute_logits(hidden_states, sampling_metadata)
747
748
# Only perform sampling in the driver worker.
855
- if not sampling_metadata.perform_sampling:
749
+ if not self.is_driver_worker:
750
return None
751
752
# Sample the next token.
@@ -860,6 +754,7 @@ class ModelRunner:
754
logits=logits,
755
sampling_metadata=sampling_metadata,
756
)
757
+
758
return output
759
760
@torch.inference_mode()
vllm/worker/neuron_model_runner.py
+12
-107
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Optional, Tuple
1
+from typing import List, Optional, Tuple
2
3
import torch
4
from torch import nn
@@ -8,10 +8,8 @@ from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
8
from vllm.logger import init_logger
9
from vllm.model_executor import SamplingMetadata
10
from vllm.model_executor.model_loader.neuron import get_neuron_model
11
-from vllm.sampling_params import SamplingParams, SamplingType
12
-from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
13
-from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
14
- make_tensor_with_pad, maybe_expand_dim)
11
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
12
+from vllm.utils import is_pin_memory_available, make_tensor_with_pad
13
14
logger = init_logger(__name__)
15
@@ -141,106 +139,6 @@ class NeuronModelRunner:
139
140
return input_tokens, input_positions, input_block_ids
141
144
- def _prepare_sample(
145
- self,
146
- seq_group_metadata_list: List[SequenceGroupMetadata],
147
- prompt_lens: List[int],
148
- ) -> SamplingMetadata:
149
- seq_groups: List[Tuple[List[int], SamplingParams]] = []
150
- selected_token_indices: List[int] = []
151
- generators: List[torch.Generator] = []
152
- selected_token_start_idx = 0
153
- categorized_sample_indices: Dict[SamplingType,
154
- List[Tuple[int, int]]] = {
155
- t: []
156
- for t in SamplingType
157
- }
158
- categorized_sample_indices_start_idx = 0
159
- categorized_sampled_token_indices_start_idx = 0
160
-
161
- for i, seq_group_metadata in enumerate(seq_group_metadata_list):
162
- seq_ids = list(seq_group_metadata.seq_data.keys())
163
- sampling_params = seq_group_metadata.sampling_params
164
- seq_groups.append((seq_ids, sampling_params))
165
-
166
- if seq_group_metadata.is_prompt:
167
- assert len(seq_ids) == 1
168
- assert prompt_lens is not None
169
- prompt_len = prompt_lens[i]
170
- if sampling_params.prompt_logprobs is not None:
171
- # NOTE: prompt token positions do not need sample, skip
172
- categorized_sample_indices_start_idx += prompt_len - 1
173
-
174
- categorized_sample_indices[
175
- sampling_params.sampling_type].append(
176
- (categorized_sample_indices_start_idx,
177
- categorized_sampled_token_indices_start_idx))
178
- categorized_sample_indices_start_idx += 1
179
- categorized_sampled_token_indices_start_idx += 1
180
-
181
- if sampling_params.prompt_logprobs is not None:
182
- selected_token_indices.extend(
183
- range(selected_token_start_idx,
184
- selected_token_start_idx + prompt_len - 1))
185
- selected_token_indices.append(selected_token_start_idx +
186
- prompt_len - 1)
187
- selected_token_start_idx += prompt_len
188
-
189
- if sampling_params.seed is not None:
190
- seq_group_metadata.state.generator = torch.Generator(
191
- device=self.device).manual_seed(sampling_params.seed)
192
- else:
193
- num_seqs = len(seq_ids)
194
- selected_token_indices.extend(
195
- range(selected_token_start_idx,
196
- selected_token_start_idx + num_seqs))
197
- selected_token_start_idx += num_seqs
198
-
199
- categorized_sample_indices[
200
- sampling_params.sampling_type].extend(
201
- zip(
202
- range(
203
- categorized_sample_indices_start_idx,
204
- categorized_sample_indices_start_idx +
205
- num_seqs),
206
- range(
207
- categorized_sampled_token_indices_start_idx,
208
- categorized_sampled_token_indices_start_idx +
209
- num_seqs)))
210
- categorized_sample_indices_start_idx += num_seqs
211
- categorized_sampled_token_indices_start_idx += num_seqs
212
-
213
- if sampling_params.seed is not None:
214
- generators.append(seq_group_metadata.state.generator)
215
-
216
- selected_token_indices = async_tensor_h2d(selected_token_indices,
217
- dtype=torch.long,
218
- target_device=self.device,
219
- pin_memory=self.pin_memory)
220
-
221
- categorized_sample_indices = {
222
- t: maybe_expand_dim(
223
- async_tensor_h2d(seq_ids,
224
- dtype=torch.int,
225
- target_device=self.device,
226
- pin_memory=self.pin_memory), 2, 2)
227
- for t, seq_ids in categorized_sample_indices.items()
228
- }
229
-
230
- seq_data: Dict[int, SequenceData] = {}
231
- for seq_group_metadata in seq_group_metadata_list:
232
- seq_data.update(seq_group_metadata.seq_data)
233
-
234
- sampling_metadata = SamplingMetadata(
235
- seq_groups=seq_groups,
236
- seq_data=seq_data,
237
- prompt_lens=prompt_lens,
238
- selected_token_indices=selected_token_indices,
239
- categorized_sample_indices=categorized_sample_indices,
240
- generators=generators,
241
- )
242
- return sampling_metadata
243
-
142
def prepare_input_tensors(
143
self,
144
seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -256,8 +154,15 @@ class NeuronModelRunner:
154
(input_tokens, input_positions,
155
input_block_ids) = self._prepare_decode(seq_group_metadata_list)
156
prompt_lens = []
259
- sampling_metadata = self._prepare_sample(seq_group_metadata_list,
260
- prompt_lens)
157
+ sampling_metadata = SamplingMetadata.prepare(
158
+ seq_group_metadata_list,
159
+ prompt_lens,
160
+ # subquery_lens is not needed if chunked prefill is not
161
+ # supported. Since neuron worker doesn't support chunked prefill
162
+ # just use prompt_lens instead.
163
+ prompt_lens,
164
+ self.device,
165
+ self.pin_memory)
166
167
return (input_tokens, input_positions, input_block_ids,
168
sampling_metadata)