A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[1/n] Triton sampling kernel (#3186)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
Antoni Baum committed
Mar 20, 2024 at 14:45 UTC
426ec4ec6711b4180538cd56b9f6b856e5276a1f
10 files changed
+1072
-24
tests/kernels/test_rand.py
new
+51
@@ -0,0 +1,51 @@
1
+import torch
2
+import pytest
3
+import random
4
+
5
+from vllm.model_executor.layers.ops.rand import seeded_uniform
6
+from vllm.model_executor.utils import set_random_seed
7
+
8
+
9
+@pytest.mark.parametrize("dtype",
10
+ [torch.float32, torch.float16, torch.bfloat16])
11
+@pytest.mark.parametrize("use_3d", [True, False])
12
+def test_seeded_uniform(dtype: torch.dtype, use_3d: bool):
13
+ device = "cuda"
14
+ for seed in range(512):
15
+ set_random_seed(seed)
16
+ rows = random.randint(1, 512)
17
+ cols = random.randint(1, 64000)
18
+ if use_3d:
19
+ third_dim = random.randint(2, 10)
20
+ dims = [rows, third_dim, cols]
21
+ else:
22
+ dims = [rows, cols]
23
+ seeds = torch.randint(torch.iinfo(torch.long).min,
24
+ torch.iinfo(torch.long).max, (rows, ),
25
+ device=device)
26
+
27
+ # Test that the same seed produces the same output
28
+ out = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
29
+ out2 = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
30
+ torch.testing.assert_close(out, out2)
31
+ # del to save memory
32
+ del out2
33
+
34
+ out3 = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
35
+ torch.testing.assert_close(out, out3)
36
+ # del to save memory
37
+ del out3
38
+
39
+ # Initialize out tensor with garbage to ensure that it is overwritten
40
+ out_with_tensor = seeded_uniform(
41
+ *dims,
42
+ out=torch.full(
43
+ (*dims, ),
44
+ -1,
45
+ dtype=dtype,
46
+ device=device,
47
+ ),
48
+ seeds=seeds,
49
+ dtype=dtype,
50
+ )
51
+ torch.testing.assert_close(out, out_with_tensor)
tests/kernels/test_sampler.py
new
+196
@@ -0,0 +1,196 @@
1
+import gc
2
+
3
+import torch
4
+import pytest
5
+import triton
6
+import triton.language as tl
7
+
8
+from vllm.model_executor.layers.ops.sample import (
9
+ _uniform_to_exponential, sample, get_num_triton_sampler_splits,
10
+ MAX_TRITON_N_COLS)
11
+from vllm.model_executor.utils import set_random_seed
12
+from vllm.model_executor.sampling_metadata import SamplingTensors
13
+
14
+SINGLE_SPLIT_VOCAB_SIZE = 32000 # llama/mistral/mixtral vocab size
15
+MULTI_SPLIT_VOCAB_SIZE = MAX_TRITON_N_COLS + 100
16
+
17
+
18
+@pytest.fixture(autouse=True)
19
+def _cleanup():
20
+ yield
21
+ gc.collect()
22
+ torch.cuda.empty_cache()
23
+
24
+
25
+@triton.jit
26
+def _uniform_to_exponential_kernel(input, output, n: tl.constexpr):
27
+ idx = tl.arange(0, n)
28
+ x = tl.load(input + idx)
29
+ y = _uniform_to_exponential(x)
30
+ tl.store(output + idx, y)
31
+
32
+
33
+def test_uniform_to_exponential():
34
+ """Test that we can convert uniform to exponential without div by 0."""
35
+ input = torch.tensor([0.0, 1.0 - torch.finfo(torch.float32).eps],
36
+ dtype=torch.float32,
37
+ device="cuda")
38
+ output = torch.zeros(input.shape, dtype=torch.float32, device="cuda")
39
+ _uniform_to_exponential_kernel[(1, )](input, output, 2)
40
+ assert torch.all(torch.isfinite(output))
41
+ assert torch.all(output > 0)
42
+ assert torch.all(torch.isfinite(torch.full_like(output, 1.0) / output))
43
+
44
+
45
+@pytest.mark.parametrize("random_sampling", [True, False, "mixed"])
46
+@pytest.mark.parametrize("max_best_of", [1, 2, 3, 4, 5])
47
+@pytest.mark.parametrize("modify_greedy_probs", [True, False])
48
+@pytest.mark.parametrize("seed", [1337])
49
+@pytest.mark.parametrize("vocab_size",
50
+ [SINGLE_SPLIT_VOCAB_SIZE, MULTI_SPLIT_VOCAB_SIZE])
51
+@pytest.mark.parametrize("save_logprobs", [True, False])
52
+def test_sample_decoding_only(random_sampling, max_best_of,
53
+ modify_greedy_probs, seed, vocab_size,
54
+ save_logprobs):
55
+ set_random_seed(seed)
56
+ bs = 8
57
+ probs = torch.zeros((bs, vocab_size), dtype=torch.float32, device="cuda")
58
+ for i in range(bs):
59
+ probs[i, i * (vocab_size // bs)] = 1.0
60
+ logprobs = torch.rand_like(probs)
61
+ sample_indices = torch.arange(bs, dtype=torch.long, device="cuda")
62
+ n_splits = get_num_triton_sampler_splits(probs.shape[1])
63
+ if random_sampling == "mixed":
64
+ random_sampling_mask = (torch.rand(
65
+ (1, bs), device="cuda") < 0.5).expand(n_splits, bs)
66
+ elif random_sampling:
67
+ random_sampling_mask = torch.ones((n_splits, bs),
68
+ dtype=torch.bool,
69
+ device="cuda")
70
+ else:
71
+ random_sampling_mask = torch.zeros((n_splits, bs),
72
+ dtype=torch.bool,
73
+ device="cuda")
74
+
75
+ seeds = torch.randint(1,
76
+ torch.iinfo(torch.long).max, (n_splits, bs),
77
+ device="cuda").mul_(random_sampling_mask)
78
+ sampled_tokens, sampled_logprobs, sampled_modified_probs = sample(
79
+ probs=probs,
80
+ logprobs=logprobs,
81
+ sample_indices=sample_indices,
82
+ seeds=seeds,
83
+ max_best_of=max_best_of,
84
+ modify_greedy_probs=modify_greedy_probs,
85
+ save_logprobs=save_logprobs,
86
+ _save_modified_probs=True)
87
+ assert sampled_tokens.shape == (bs, max_best_of)
88
+ for i in range(bs):
89
+ assert torch.all(sampled_tokens[i] == i * (vocab_size // bs))
90
+ request_uses_random_sampling = random_sampling_mask[0, i]
91
+ if modify_greedy_probs and not request_uses_random_sampling:
92
+ # If we are modifying greedy probs and the request is greedy,
93
+ # we want to make sure the probs tensor is modified in place
94
+ assert torch.allclose(
95
+ probs[i][sampled_tokens[i]],
96
+ torch.full_like(probs[i][sampled_tokens[i]], 1.0))
97
+ assert torch.sum(probs[i]) == 1.0
98
+ assert torch.allclose(
99
+ sampled_modified_probs[i][0],
100
+ torch.full_like(sampled_modified_probs[i][0], 1.0))
101
+ elif request_uses_random_sampling:
102
+ # If the request is random, we want to make sure
103
+ # sampled_modified_probs tensor has noise added
104
+ # (and thus is different from probs tensor)
105
+ assert not torch.allclose(sampled_modified_probs[i][0],
106
+ probs[i][sampled_tokens[i]])
107
+ elif not request_uses_random_sampling:
108
+ # If the request is greedy and we are not modifying greedy probs,
109
+ # we want to make sure sampled_modified_probs tensor is the same as
110
+ # the probs tensor.
111
+ assert torch.allclose(sampled_modified_probs[i][0],
112
+ probs[i][sampled_tokens[i]])
113
+
114
+ if save_logprobs:
115
+ assert sampled_logprobs.shape == (bs, max_best_of)
116
+ for i in range(bs):
117
+ for best_of in range(max_best_of):
118
+ assert torch.all(sampled_logprobs[i] == logprobs[i][
119
+ sampled_tokens[i, best_of]])
120
+ else:
121
+ assert sampled_logprobs is None
122
+
123
+
124
+@pytest.mark.parametrize("random_sampling", [True, False, "mixed"])
125
+@pytest.mark.parametrize("max_best_of", [1, 2, 3, 4, 5])
126
+@pytest.mark.parametrize("modify_greedy_probs", [True, False])
127
+@pytest.mark.parametrize("seed", [1337])
128
+@pytest.mark.parametrize("vocab_size",
129
+ [SINGLE_SPLIT_VOCAB_SIZE, MULTI_SPLIT_VOCAB_SIZE])
130
+def test_sample_prompt_logprobs(random_sampling, max_best_of,
131
+ modify_greedy_probs, seed, vocab_size):
132
+ set_random_seed(seed)
133
+ prompt_sizes = [16, 32, 64, 128] * 2
134
+ samples = 8
135
+ bs = samples + sum(prompt_sizes)
136
+ probs = torch.zeros((bs, vocab_size), dtype=torch.float32, device="cuda")
137
+ for i in range(bs):
138
+ probs[i, i * (vocab_size // bs)] = 1.0
139
+ logprobs = torch.rand_like(probs)
140
+ sample_indices = torch.tensor(prompt_sizes,
141
+ dtype=torch.long,
142
+ device="cuda").cumsum_(0)
143
+ n_splits = get_num_triton_sampler_splits(probs.shape[1])
144
+ if random_sampling == "mixed":
145
+ random_sampling_mask = torch.rand(
146
+ (n_splits, samples), device="cuda") < 0.5
147
+ elif random_sampling:
148
+ random_sampling_mask = torch.ones((n_splits, samples),
149
+ dtype=torch.bool,
150
+ device="cuda")
151
+ else:
152
+ random_sampling_mask = torch.zeros((n_splits, samples),
153
+ dtype=torch.bool,
154
+ device="cuda")
155
+
156
+ seeds = torch.randint(1,
157
+ torch.iinfo(torch.long).max, (n_splits, samples),
158
+ device="cuda").mul_(random_sampling_mask)
159
+ sampled_tokens, sampled_logprobs, _ = sample(
160
+ probs=probs,
161
+ logprobs=logprobs,
162
+ sample_indices=sample_indices,
163
+ seeds=seeds,
164
+ max_best_of=max_best_of,
165
+ modify_greedy_probs=modify_greedy_probs,
166
+ save_logprobs=True)
167
+ assert sampled_tokens.shape == (samples, max_best_of)
168
+ assert sampled_logprobs.shape == (samples, max_best_of)
169
+ for i, t in enumerate(sample_indices):
170
+ assert torch.all(sampled_tokens[i] == t * (vocab_size // bs))
171
+ for best_of in range(max_best_of):
172
+ assert torch.all(sampled_logprobs[i] == logprobs[sample_indices[i]]
173
+ [sampled_tokens[i, best_of]])
174
+
175
+
176
+@pytest.mark.parametrize("seed", list(range(16)))
177
+def test_get_sequence_seeds(seed):
178
+ """Ensure that we get a different child seed from base
179
+ seed + extra entropy"""
180
+ starting_seed = seed
181
+ seq_seed = None
182
+ extra_entropy = 1
183
+ for i in range(512):
184
+ new_seq_seed = SamplingTensors._get_sequence_seeds(starting_seed,
185
+ i,
186
+ seeds_to_generate=1,
187
+ is_greedy=False)[0]
188
+ new_seq_seed_extra_entropy = SamplingTensors._get_sequence_seeds(
189
+ starting_seed,
190
+ i,
191
+ extra_entropy,
192
+ seeds_to_generate=1,
193
+ is_greedy=False)[0]
194
+ assert new_seq_seed_extra_entropy != new_seq_seed
195
+ assert seq_seed != new_seq_seed
196
+ seq_seed = new_seq_seed
tests/samplers/test_sampler.py
+3
-3
@@ -302,11 +302,11 @@ def test_sampler_logits_processors(seed: int, device: str):
302
batch_size = random.randint(1, 256)
303
input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
304
305
- # This sample logits processor gives infinite score to the i-th token,
305
+ # This sample logits processor gives maximum score to the i-th token,
306
# where i is the length of the input sequence.
307
# We therefore expect the output token sequence to be [0, 1, 2, ...]
308
def pick_ith(token_ids, logits):
309
- logits[len(token_ids)] = float("inf")
309
+ logits[len(token_ids)] = torch.finfo(logits.dtype).max
310
return logits
311
312
seq_group_metadata_list = []
@@ -385,7 +385,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
385
386
sample_probs = None
387
388
- def mock_sample(probs, logprobs, sampling_metadata):
388
+ def mock_sample(probs, *args, **kwargs):
389
nonlocal sample_probs
390
sample_probs = probs
391
return [[prob.topk(1, dim=-1).indices.tolist(), [0]] for prob in probs]
vllm/model_executor/layers/ops/__init__.py
vllm/model_executor/layers/ops/rand.py
new
+157
@@ -0,0 +1,157 @@
1
+import torch
2
+import triton
3
+import triton.language as tl
4
+
5
+from typing import Optional, Union
6
+
7
+
8
+def seeded_uniform(
9
+ *size,
10
+ seeds: torch.Tensor,
11
+ out: Optional[torch.Tensor] = None,
12
+ dtype: Optional[torch.dtype] = None,
13
+ device: Optional[Union[torch.device, str]] = None,
14
+ pin_memory: Optional[bool] = False,
15
+) -> torch.Tensor:
16
+ """Similar to torch.rand, but allows for seeds to be set per row.
17
+
18
+ seeds must be a 1d tensor. The output tensor may be 1d, 2d, or 3d.
19
+ If it is 3d, the additional seeds needed will be derived automatically
20
+ in a deterministic fashion:
21
+ [
22
+ row 0: [columns_with_seed_0], [columns_with_seed0^1], ...
23
+ ]
24
+ """
25
+ n_dims = len(size)
26
+
27
+ if n_dims > 3:
28
+ raise ValueError("seeded_uniform only supports up to 3D tensors")
29
+
30
+ if out is None:
31
+ out = torch.empty(*size,
32
+ dtype=dtype,
33
+ device=device,
34
+ pin_memory=pin_memory)
35
+ elif out.shape != size:
36
+ raise ValueError("shape of out and size must be the same")
37
+
38
+ if n_dims == 3:
39
+ n_rows, n_3d, n_cols = out.shape
40
+ stride_row = out.stride(0)
41
+ stride_3d = out.stride(1)
42
+ elif n_dims == 2:
43
+ n_rows, n_cols = out.shape
44
+ n_3d = 1
45
+ stride_row = out.stride(0)
46
+ stride_3d = 1
47
+ else:
48
+ n_cols = out.shape[0]
49
+ n_rows = 1
50
+ n_3d = 1
51
+ stride_row = 1
52
+ stride_3d = 1
53
+
54
+ if seeds.ndim != 1:
55
+ raise ValueError("seeds must be a 1D tensor")
56
+
57
+ if seeds.numel() != n_rows:
58
+ raise ValueError(
59
+ "seeds must have the same number of elements as out has rows")
60
+
61
+ # The philox PRNG Triton uses generates 4 random numbers at once.
62
+ # Therefore, the most efficient use of it is to divide the
63
+ # block size by 4, and then save the generated random numbers to
64
+ # each of the 4 slices of the tensor.
65
+ full_block_size = triton.next_power_of_2(n_cols)
66
+ philox_block_size = max(full_block_size // 4, 1)
67
+ n_slices = full_block_size // philox_block_size
68
+ num_warps = 4
69
+ # Manual tuning. This seems to give best performance on A100 for
70
+ # simple kernels like this.
71
+ if philox_block_size >= 8192:
72
+ num_warps = 32
73
+ elif philox_block_size >= 4096:
74
+ num_warps = 16
75
+ elif philox_block_size >= 2048:
76
+ num_warps = 8
77
+
78
+ _seeded_uniform_triton[(n_rows, n_3d)](
79
+ out,
80
+ seeds,
81
+ stride_row,
82
+ stride_3d,
83
+ seeds.stride(0),
84
+ n_rows,
85
+ n_3d,
86
+ n_cols,
87
+ n_slices=n_slices,
88
+ num_warps=num_warps,
89
+ block_size=philox_block_size,
90
+ )
91
+ return out
92
+
93
+
94
+@triton.jit
95
+def _seeded_uniform_triton(
96
+ out_ptr: torch.Tensor,
97
+ seed_ptr: torch.Tensor,
98
+ out_row_stride: int,
99
+ out_3d_stride: int,
100
+ seed_row_stride: int,
101
+ n_rows: int,
102
+ n_3d: int,
103
+ n_cols: int,
104
+ n_slices: tl.constexpr,
105
+ block_size: tl.constexpr,
106
+):
107
+ """
108
+ Generate a random float32 number in [0, 1) for each element in the output
109
+ tensor. The random numbers in a row generated using the seed for that row.
110
+
111
+ Args:
112
+ out_ptr: The output tensor.
113
+ seed_ptr: The per-row seeds to use for random number generation.
114
+ out_row_stride: The stride between rows of the output tensor.
115
+ out_3d_stride: The stride between 3D slices of the output tensor.
116
+ seed_row_stride: The stride between rows of the seed tensor.
117
+ n_rows: The number of rows in the output tensor.
118
+ n_3d: The size of second dimension of the output tensor,
119
+ if output tensor is 3D.
120
+ n_cols: The number of columns in the output tensor.
121
+ n_slices: The number of philox outputs to use.
122
+ """
123
+ tl.static_assert(n_slices > 0 and n_slices <= 4, "0 < n_slices <= 4")
124
+
125
+ # Get the row index.
126
+ row_idx = tl.program_id(axis=0)
127
+ three_d_idx = tl.program_id(axis=1)
128
+
129
+ philox_offsets = tl.arange(0, block_size)
130
+ # Get the seed for the current element.
131
+ seed = tl.load(seed_ptr + row_idx * seed_row_stride)
132
+ if three_d_idx > 0:
133
+ seed ^= three_d_idx
134
+ # Generate random numbers in [0, 1).
135
+ out1, out2, out3, out4 = tl.rand4x(seed, philox_offsets)
136
+
137
+ output_row_start_ptr = (out_ptr + row_idx * out_row_stride +
138
+ three_d_idx * out_3d_stride)
139
+ out1_offsets = philox_offsets
140
+ tl.store(output_row_start_ptr + out1_offsets,
141
+ out1,
142
+ mask=out1_offsets < n_cols)
143
+ if n_slices > 1:
144
+ out2_offsets = tl.arange(block_size, block_size * 2)
145
+ tl.store(output_row_start_ptr + out2_offsets,
146
+ out2,
147
+ mask=out2_offsets < n_cols)
148
+ if n_slices > 2:
149
+ out3_offsets = tl.arange(block_size * 2, block_size * 3)
150
+ tl.store(output_row_start_ptr + out3_offsets,
151
+ out3,
152
+ mask=out3_offsets < n_cols)
153
+ if n_slices > 3:
154
+ out4_offsets = tl.arange(block_size * 3, block_size * 4)
155
+ tl.store(output_row_start_ptr + out4_offsets,
156
+ out4,
157
+ mask=out4_offsets < n_cols)
vllm/model_executor/layers/ops/sample.py
new
+405
@@ -0,0 +1,405 @@
1
+import math
2
+from typing import Tuple, Optional
3
+
4
+import torch
5
+import triton
6
+import triton.language as tl
7
+
8
+from vllm.model_executor.layers.ops.rand import seeded_uniform
9
+
10
+_EPS = 1e-6
11
+
12
+# This is a hardcoded limit in Triton (max block size).
13
+MAX_TRITON_N_COLS = 131072
14
+
15
+
16
+def get_num_triton_sampler_splits(n_cols: int) -> int:
17
+ """Get the number of splits to use for Triton sampling.
18
+
19
+ Triton has a limit on the number of columns it can handle, so we need to
20
+ split the tensor and call the kernel multiple times if it's too large.
21
+ """
22
+ return math.ceil(n_cols / MAX_TRITON_N_COLS)
23
+
24
+
25
+def _multi_split_sample(
26
+ probs: torch.Tensor,
27
+ seeds: torch.Tensor,
28
+ n_splits: int,
29
+ sampled_tokens_size: Tuple[int, int],
30
+ sampled_logprobs_size: Tuple[int, int],
31
+ sample_indices: torch.Tensor,
32
+ *,
33
+ logprobs: Optional[torch.Tensor] = None,
34
+ modify_greedy_probs: bool = False,
35
+ save_logprobs: bool = False,
36
+):
37
+ """Sample tokens where vocab size is split into multiple parts
38
+ (too large for Triton otherwise)."""
39
+ assert seeds.ndim == 2 and seeds.shape[0] == n_splits
40
+ split_probs = probs.tensor_split(n_splits, 1)
41
+ split_logprobs = logprobs.tensor_split(n_splits, 1)
42
+ sampled_tokens_tmp = [
43
+ torch.empty(sampled_tokens_size, dtype=torch.long, device=probs.device)
44
+ for _ in range(n_splits)
45
+ ]
46
+ sampled_logprobs_tmp = [
47
+ torch.empty(sampled_logprobs_size,
48
+ dtype=probs.dtype,
49
+ device=probs.device) for _ in range(n_splits)
50
+ ]
51
+ # We are purposefuly using sampled_tokens_size as we need to always
52
+ # save modified probs in this case.
53
+ sampled_modified_probs_tmp = [
54
+ torch.empty(sampled_tokens_size,
55
+ dtype=probs.dtype,
56
+ device=probs.device) for _ in range(n_splits)
57
+ ]
58
+ for i in range(n_splits):
59
+ n_samples = sample_indices.shape[0]
60
+ n_cols = split_probs[i].shape[1]
61
+ n_best = sampled_tokens_tmp[i].shape[1]
62
+ uniform_noise = seeded_uniform(n_samples,
63
+ n_best,
64
+ n_cols,
65
+ seeds=seeds[i].flatten(),
66
+ device=split_probs[i].device,
67
+ dtype=split_probs[i].dtype)
68
+ # TODO(yard1): See if we can remove the contiguous() calls.
69
+ # Will need kernel support.
70
+ _sample(
71
+ split_probs[i].contiguous(),
72
+ split_logprobs[i].contiguous(),
73
+ sample_indices,
74
+ sampled_tokens_tmp[i],
75
+ sampled_logprobs_tmp[i],
76
+ sampled_modified_probs_tmp[i],
77
+ seeds[i],
78
+ uniform_noise,
79
+ modify_greedy_probs=False,
80
+ save_logprobs=save_logprobs,
81
+ save_modified_probs=True,
82
+ )
83
+ if i > 0:
84
+ # Add offset to sampled tokens
85
+ sampled_tokens_tmp[i].add_(i * split_probs[i - 1].shape[1])
86
+ sampled_tokens = torch.stack(sampled_tokens_tmp)
87
+ sampled_modified_probs = torch.stack(sampled_modified_probs_tmp)
88
+ # Reduce the results from the splits.
89
+ sampled_modified_probs, indices = torch.max(sampled_modified_probs,
90
+ dim=0,
91
+ keepdim=True)
92
+ sampled_tokens = sampled_tokens.gather(0, indices).squeeze(0)
93
+ if save_logprobs:
94
+ sampled_logprobs = torch.stack(sampled_logprobs_tmp)
95
+ sampled_logprobs = sampled_logprobs.gather(0, indices).squeeze(0)
96
+ else:
97
+ sampled_logprobs = None
98
+ sampled_modified_probs = sampled_modified_probs.squeeze(0)
99
+
100
+ if modify_greedy_probs:
101
+ # We need to modify the greedy probs for the sampled tokens.
102
+ # We can't do this in the kernel as we need to know the
103
+ # sampled tokens.
104
+ probs.fill_(0.0)
105
+ probs.scatter_(1, sampled_tokens, 1.0)
106
+
107
+ return (sampled_tokens, sampled_logprobs, sampled_modified_probs)
108
+
109
+
110
+def sample(
111
+ probs: torch.Tensor,
112
+ seeds: torch.Tensor,
113
+ *,
114
+ max_best_of: int = 1,
115
+ sample_indices: Optional[torch.Tensor] = None,
116
+ logprobs: Optional[torch.Tensor] = None,
117
+ modify_greedy_probs: bool = False,
118
+ save_logprobs: bool = False,
119
+ _save_modified_probs: bool = False, # pylint: disable=invalid-name
120
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]:
121
+ """Sample tokens from probs. with per-sequence seeds.
122
+
123
+ Can sample from a subset of sequences through sample_indices.
124
+
125
+ Args:
126
+ probs: Probabilities to sample from.
127
+ shape = [batch_size, vocab_size]
128
+ seeds: Per-sequence seed values.
129
+ shape = [n, math.ceil(vocab_size / MAX_TRITON_N_COLS)]
130
+ max_best_of: Number of samples to generate per sequence.
131
+ Sequence seed will be incremented by 1 each time.
132
+ sample_indices: Indices of sequences to sample from.
133
+ If not provided, will sample from all sequences.
134
+ shape = [n]
135
+ logprobs: Log-probabilities of the sampled tokens.
136
+ Only used for saving the logprobs if save_logprobs is True.
137
+ shape = [batch_size, vocab_size]
138
+ modify_greedy_probs: Whether to modify the greedy probabilities
139
+ for speculative sampling (sampled token = 1.0,
140
+ everything else = 0.0).
141
+ save_logprobs: Whether to save the log-probabilities of the
142
+ sampled tokens to a tensor.
143
+ _save_modified_probs: Whether to save the modified probabilities
144
+ (including gumbel noise) of the sampled tokens to a tensor.
145
+ DOES NOT include the modification done by modify_greedy_probs
146
+ (because we want to use the unmodified probs to pick the best
147
+ split in case of multi-split sampling).
148
+ This is exposed only for testing.
149
+
150
+ Returns:
151
+ sampled_tokens: shape = [n, max_best_of]
152
+ sampled_logprobs: shape = [n, max_best_of] if save_logprobs else None
153
+ sampled_modified_probs: shape = [n, max_best_of]
154
+ if save_modified_probs else None
155
+ """
156
+ if sample_indices is None:
157
+ sample_indices = torch.arange(0, probs.shape[0], device=probs.device)
158
+
159
+ sampled_tokens_size = (sample_indices.size(0), max_best_of)
160
+ if save_logprobs:
161
+ if logprobs is None:
162
+ raise ValueError(
163
+ "logprobs tensor must be provided if save_logprobs is True")
164
+ sampled_logprobs_size = sampled_tokens_size
165
+ else:
166
+ # Empty tensors to invoke the kernel
167
+ sampled_logprobs_size = (0, 0)
168
+ logprobs = probs
169
+
170
+ if _save_modified_probs:
171
+ sampled_modified_probs_size = sampled_tokens_size
172
+ else:
173
+ # Empty tensors to invoke the kernel
174
+ sampled_modified_probs_size = (0, 0)
175
+
176
+ # If the number of columns in probs is too large for Triton to handle,
177
+ # we split the tensor and sample from each split separately, and then
178
+ # do an argmax+gather to combine the results.
179
+ n_splits = get_num_triton_sampler_splits(probs.shape[1])
180
+ if n_splits > 1:
181
+ (sampled_tokens, sampled_logprobs,
182
+ sampled_modified_probs) = _multi_split_sample(
183
+ probs,
184
+ seeds,
185
+ n_splits,
186
+ sampled_tokens_size,
187
+ sampled_logprobs_size,
188
+ sample_indices,
189
+ logprobs=logprobs,
190
+ modify_greedy_probs=modify_greedy_probs,
191
+ save_logprobs=save_logprobs)
192
+ else:
193
+ sampled_tokens = torch.empty(sampled_tokens_size,
194
+ dtype=torch.long,
195
+ device=probs.device)
196
+ sampled_logprobs = torch.empty(sampled_logprobs_size,
197
+ dtype=probs.dtype,
198
+ device=probs.device)
199
+ sampled_modified_probs = torch.empty(sampled_modified_probs_size,
200
+ dtype=probs.dtype,
201
+ device=probs.device)
202
+ n_samples = sample_indices.shape[0]
203
+ n_cols = probs.shape[1]
204
+ uniform_noise = seeded_uniform(n_samples,
205
+ max_best_of,
206
+ n_cols,
207
+ seeds=seeds.flatten(),
208
+ device=probs.device,
209
+ dtype=probs.dtype)
210
+
211
+ _sample(
212
+ probs,
213
+ logprobs,
214
+ sample_indices,
215
+ sampled_tokens,
216
+ sampled_logprobs,
217
+ sampled_modified_probs,
218
+ seeds,
219
+ uniform_noise,
220
+ modify_greedy_probs=modify_greedy_probs,
221
+ save_logprobs=save_logprobs,
222
+ save_modified_probs=_save_modified_probs,
223
+ )
224
+ return (sampled_tokens, sampled_logprobs if save_logprobs else None,
225
+ sampled_modified_probs if _save_modified_probs else None)
226
+
227
+
228
+def _sample(probs: torch.Tensor,
229
+ logprobs: torch.Tensor,
230
+ sample_indices: torch.Tensor,
231
+ output_samples: torch.Tensor,
232
+ output_logprobs: torch.Tensor,
233
+ output_modified_probs: torch.Tensor,
234
+ seeds: torch.Tensor,
235
+ uniform_noise: torch.Tensor,
236
+ *,
237
+ modify_greedy_probs: bool = False,
238
+ save_logprobs: bool = True,
239
+ save_modified_probs: bool = False) -> torch.Tensor:
240
+ """Sample tokens from probs.
241
+
242
+ Args:
243
+ probs [batch_size, vocab_size]: probs to sample from.
244
+ logprobs [batch_size, vocab_size]: logprobs (used when
245
+ save_logprobsis True).
246
+ sample_indices [n]: Indices of the samples to use for each row of probs.
247
+ output_samples [n, n_best]: Output tensor to store samples in.
248
+ output_logprobs [n, n_best]: Output tensor to store logprobs in.
249
+ output_modified_probs [n, n_best]: Output tensor to store
250
+ probs of chosen tokens in (modified with noise).
251
+ seeds [n]: Seeds to use for sampling. If the seed is 0, we use
252
+ greedy sampling. Note this is ONLY used for determining
253
+ whether to use random sampling or not. The actual random
254
+ noise should be passed as uniform_noise.
255
+ uniform_noise [batch_size, n_best, vocab_size]: Uniform
256
+ noise to use for random sampling (will be converted
257
+ to exponential gumbel noise by the kernel).
258
+ modify_greedy_probs: If True, we modify the probs tensor in-place
259
+ to encode the sampling method used for each row. This is used
260
+ in speculative decoding. Only applies in greedy decoding.
261
+ save_logprobs: If True, we save the logprobs of the sampled tokens
262
+ in the output_logprobs tensor.
263
+ save_modified_probs: If True, we save the modified probs (with noise)
264
+ of the sampled tokens in the output_modified_probs tensor.
265
+ DOES NOT include the modification done by modify_greedy_probs
266
+ (because we want to use the unmodified probs to pick the best
267
+ split in case of multi-split sampling).
268
+ """
269
+ n_samples = sample_indices.shape[0]
270
+ n_cols = probs.shape[1]
271
+ n_best = output_samples.shape[1] if len(output_samples.shape) > 1 else 1
272
+
273
+ # The block size is the smallest power of two greater than the number of
274
+ # columns in probs
275
+ block_size = triton.next_power_of_2(n_cols)
276
+ num_warps = 4
277
+ # Manual tuning. This seems to give best performance on A100 for
278
+ # simple kernels like this.
279
+ if block_size >= 8192:
280
+ num_warps = 32
281
+ elif block_size >= 4096:
282
+ num_warps = 16
283
+ elif block_size >= 2048:
284
+ num_warps = 8
285
+
286
+ # Enqueue kernel. The 1D launch grid is simple: we have one kernel
287
+ # instance per row of the probs matrix
288
+ _sample_triton[(n_samples, n_best)](
289
+ sample_indices,
290
+ output_samples,
291
+ output_logprobs,
292
+ output_modified_probs,
293
+ probs,
294
+ logprobs,
295
+ seeds,
296
+ uniform_noise,
297
+ output_samples.stride(0),
298
+ probs.stride(0),
299
+ uniform_noise.stride(0),
300
+ uniform_noise.stride(1) if n_best > 1 else 1,
301
+ n_samples,
302
+ n_cols,
303
+ n_best,
304
+ num_warps=num_warps,
305
+ block_size=block_size,
306
+ modify_greedy_probs=modify_greedy_probs,
307
+ save_logprobs=save_logprobs,
308
+ save_modified_probs=save_modified_probs,
309
+ )
310
+ return output_samples, output_logprobs, output_modified_probs
311
+
312
+
313
+@triton.jit
314
+def _uniform_to_exponential(uniform_noise):
315
+ """Convert uniform samples to exponential samples."""
316
+ # tl.rand returns values in [0, 1), so we clamp lower bound
317
+ # to _EPS to avoid log(0) and thus division by 0 later
318
+ lb = tl.full(uniform_noise.shape, _EPS, uniform_noise.dtype)
319
+ uniform_noise = tl.maximum(uniform_noise, lb)
320
+ # Use the inversion method to turn uniform samples
321
+ # into exponential samples
322
+ exponential_noise = -tl.log(uniform_noise)
323
+ return exponential_noise
324
+
325
+
326
+@triton.jit
327
+def _sample_triton(
328
+ sample_indices_ptr: torch.Tensor, output_ptr: torch.Tensor,
329
+ output_logprobs_ptr: torch.Tensor,
330
+ output_modified_probs_ptr: torch.Tensor, probs_ptr: torch.Tensor,
331
+ logprobs_ptr: torch.Tensor, seeds_ptr: torch.Tensor,
332
+ uniform_noise_ptr: torch.Tensor, output_row_stride: int,
333
+ probs_row_stride: int, uniform_noise_row_stride: int,
334
+ uniform_noise_best_stride: int, n_samples: int, n_cols: int,
335
+ n_best: int, block_size: tl.constexpr,
336
+ modify_greedy_probs: tl.constexpr, save_logprobs: tl.constexpr,
337
+ save_modified_probs: tl.constexpr):
338
+ # The rows are independent, so we parallelize across those
339
+ sample_idx = tl.program_id(0)
340
+ best_idx = tl.program_id(1)
341
+
342
+ # Load the row index from DRAM
343
+ row_idx = tl.load(sample_indices_ptr + sample_idx)
344
+ seed = tl.load(seeds_ptr + sample_idx)
345
+ uses_random_sampling = seed != 0
346
+
347
+ # The stride represents how much we need to increase the
348
+ # pointer to advance 1 row
349
+ row_start_ptr = probs_ptr + row_idx * probs_row_stride
350
+
351
+ # The block size is the next power of two greater than n_cols,
352
+ # so we can fit each row in a single block
353
+ col_offsets = tl.arange(0, block_size)
354
+
355
+ # Load the row into SRAM, using a mask since block_size may be > than n_cols
356
+ row = tl.load(row_start_ptr + col_offsets,
357
+ mask=col_offsets < n_cols,
358
+ other=float("-inf"))
359
+
360
+ if uses_random_sampling:
361
+ uniform_noise_start_ptr = (uniform_noise_ptr +
362
+ sample_idx * uniform_noise_row_stride +
363
+ best_idx * uniform_noise_best_stride)
364
+ uniform_noise = tl.load(uniform_noise_start_ptr + col_offsets,
365
+ mask=col_offsets < n_cols,
366
+ other=0.5)
367
+ exponential_noise = _uniform_to_exponential(uniform_noise)
368
+ row /= exponential_noise
369
+
370
+ sampled_value, sampled_token = tl.max(row, axis=0, return_indices=True)
371
+ # clamp sampled token to n_cols - 1
372
+ # this should not be necessary, but we do it
373
+ # just in case
374
+ if sampled_token >= n_cols:
375
+ sampled_token = n_cols - 1
376
+ # Write back output to DRAM
377
+ output_row_start_ptr = (output_ptr + sample_idx * output_row_stride +
378
+ best_idx)
379
+ tl.store(output_row_start_ptr, sampled_token)
380
+
381
+ if modify_greedy_probs: # noqa
382
+ if not uses_random_sampling:
383
+ # Set the probability of the sampled token to 1, all other
384
+ # tokens to zero. This is used in speculative decoding where
385
+ # the sampling method must be encoded within the sampled
386
+ # probability distributions.
387
+ row = tl.where(col_offsets == sampled_token, 1.0, 0.0)
388
+ tl.store(row_start_ptr + col_offsets,
389
+ row,
390
+ mask=col_offsets < n_cols)
391
+
392
+ if save_modified_probs:
393
+ output_row_start_ptr = (output_modified_probs_ptr +
394
+ sample_idx * output_row_stride + best_idx)
395
+ tl.store(output_row_start_ptr, sampled_value)
396
+
397
+ if save_logprobs:
398
+ # Load the row into SRAM, using a mask since block_size
399
+ # may be > than n_cols
400
+ sampled_logprob = tl.load(logprobs_ptr + row_idx * probs_row_stride +
401
+ sampled_token)
402
+ # Write back output to DRAM
403
+ output_row_start_ptr = (output_logprobs_ptr +
404
+ sample_idx * output_row_stride + best_idx)
405
+ tl.store(output_row_start_ptr, sampled_logprob)
vllm/model_executor/layers/sampler.py
+103
-6
@@ -12,6 +12,7 @@ from vllm.sampling_params import SamplingParams, SamplingType
12
from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
13
SamplerOutput, SequenceData, SequenceGroupOutput,
14
SequenceOutput)
15
+from vllm.model_executor.layers.ops.sample import (sample as sample_triton)
16
from vllm.utils import is_neuron
17
18
@@ -114,7 +115,8 @@ class Sampler(nn.Module):
115
logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
116
117
# Sample the next tokens.
117
- sample_results = _sample(probs, logprobs, sampling_metadata)
118
+ sample_results = _sample(probs, logprobs, sampling_metadata,
119
+ sampling_tensors)
120
# Get the logprobs query results.
121
prompt_logprobs, sample_logprobs = _get_logprobs(
122
logprobs, sampling_metadata, sample_results)
@@ -375,7 +377,7 @@ def _multinomial(
377
return probs.div_(q).argmax(dim=1).view(-1, num_samples)
378
379
378
-def _sample(
380
+def _sample_with_torch(
381
probs: torch.Tensor,
382
logprobs: torch.Tensor,
383
sampling_metadata: SamplingMetadata,
@@ -394,7 +396,7 @@ def _sample(
396
# Counterintiutively, having two loops here is actually faster.
397
# The first loop can run without waiting on GPU<->CPU sync.
398
for sampling_type in SamplingType:
397
- sample_indices = categorized_sample_indices[sampling_type]
399
+ sample_indices = categorized_sample_indices[sampling_type][:, 0]
400
num_tokens = len(sample_indices)
401
if num_tokens == 0:
402
continue
@@ -407,17 +409,19 @@ def _sample(
409
greedy_samples = torch.argmax(logprobs[sample_indices.long()],
410
dim=-1)
411
elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
410
- max_best_of = 1
412
+ max_best_of_in_batch = 1
413
for seq_group, is_prompt in zip(seq_groups, is_prompts):
414
if is_prompt:
415
_, sampling_params = seq_group
414
- max_best_of = max(max_best_of, sampling_params.best_of)
416
+ max_best_of_in_batch = max(max_best_of_in_batch,
417
+ sampling_params.best_of)
418
seeded_args = {} if sampling_type == SamplingType.RANDOM else {
419
"seq_groups": seq_groups,
420
"generators": sampling_metadata.generators,
421
}
422
multinomial_samples[sampling_type] = _multinomial(
420
- probs[sample_indices.long()], max_best_of, **seeded_args)
423
+ probs[sample_indices.long()], max_best_of_in_batch,
424
+ **seeded_args)
425
elif sampling_type == SamplingType.BEAM:
426
beam_search_logprobs = logprobs[sample_indices]
427
else:
@@ -448,6 +452,99 @@ def _sample(
452
return sample_results
453
454
455
+def _sample_with_triton_kernel(
456
+ probs: torch.Tensor,
457
+ logprobs: torch.Tensor,
458
+ sampling_metadata: SamplingMetadata,
459
+ sampling_tensors: SamplingTensors,
460
+) -> List[Tuple[List[int], List[int]]]:
461
+ categorized_seq_group_ids = {t: [] for t in SamplingType}
462
+ categorized_sample_indices = sampling_metadata.categorized_sample_indices
463
+ for i, seq_group in enumerate(sampling_metadata.seq_groups):
464
+ _, sampling_params = seq_group
465
+ sampling_type = sampling_params.sampling_type
466
+ categorized_seq_group_ids[sampling_type].append(i)
467
+
468
+ sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
469
+ sample_metadata = {}
470
+ max_best_of_in_batch = 1
471
+
472
+ # Counterintiutively, having two loops here is actually faster.
473
+ # The first loop can run without waiting on GPU<->CPU sync.
474
+ for sampling_type in SamplingType:
475
+ sample_indices = categorized_sample_indices[sampling_type][:, 0]
476
+ sampled_token_indices = categorized_sample_indices[sampling_type][:, 1]
477
+ num_tokens = len(sample_indices)
478
+ if num_tokens == 0:
479
+ continue
480
+ seq_group_ids = categorized_seq_group_ids[sampling_type]
481
+ seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
482
+ is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
483
+ sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
484
+ is_prompts, sample_indices,
485
+ sampled_token_indices)
486
+ if sampling_type in (SamplingType.GREEDY, SamplingType.RANDOM,
487
+ SamplingType.RANDOM_SEED):
488
+ for seq_group, is_prompt in zip(seq_groups, is_prompts):
489
+ if is_prompt:
490
+ _, sampling_params = seq_group
491
+ max_best_of_in_batch = max(max_best_of_in_batch,
492
+ sampling_params.best_of)
493
+ elif sampling_type == SamplingType.BEAM:
494
+ beam_search_logprobs = logprobs[sample_indices]
495
+ else:
496
+ raise ValueError(f"Unsupported sampling type: {sampling_type}")
497
+
498
+ sampled_tokens, _, _ = sample_triton(
499
+ probs=probs,
500
+ seeds=sampling_tensors.sampling_seeds,
501
+ max_best_of=max_best_of_in_batch,
502
+ sample_indices=sampling_tensors.sample_indices,
503
+ logprobs=logprobs,
504
+ # don't save logprobs because we have logic for that below
505
+ # TODO: use this instead of the CPU-based logic below
506
+ save_logprobs=False,
507
+ )
508
+
509
+ # GPU<->CPU sync happens in the loop below.
510
+
511
+ for sampling_type in SamplingType:
512
+ if sampling_type not in sample_metadata:
513
+ continue
514
+ (seq_group_ids, seq_groups, is_prompts, sample_indices,
515
+ sampled_token_indices) = sample_metadata[sampling_type]
516
+ if sampling_type == SamplingType.GREEDY:
517
+ sample_results = _greedy_sample(
518
+ seq_groups, sampled_tokens[sampled_token_indices][:, 0])
519
+ elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
520
+ sample_results = _random_sample(
521
+ seq_groups, is_prompts, sampled_tokens[sampled_token_indices])
522
+ elif sampling_type == SamplingType.BEAM:
523
+ sample_results = _beam_search_sample(seq_groups, is_prompts,
524
+ sampling_metadata.seq_data,
525
+ beam_search_logprobs)
526
+ sample_results_dict.update(zip(seq_group_ids, sample_results))
527
+
528
+ sample_results = [
529
+ sample_results_dict[i]
530
+ for i in range(len(sampling_metadata.seq_groups))
531
+ ]
532
+ return sample_results
533
+
534
+
535
+def _sample(
536
+ probs: torch.Tensor,
537
+ logprobs: torch.Tensor,
538
+ sampling_metadata: SamplingMetadata,
539
+ sampling_tensors: SamplingTensors,
540
+) -> List[Tuple[List[int], List[int]]]:
541
+ return _sample_with_torch(probs, logprobs, sampling_metadata)
542
+
543
+ # TODO: Enable once Triton kernel & associated code is faster.
544
+ # return _sample_with_triton_kernel(probs, logprobs, sampling_metadata,
545
+ # sampling_tensors)
546
+
547
+
548
def _get_logprobs(
549
logprobs: torch.Tensor,
550
sampling_metadata: SamplingMetadata,
vllm/model_executor/sampling_metadata.py
+123
-6
@@ -2,12 +2,16 @@ from dataclasses import dataclass
2
from typing import Dict, List, Optional, Tuple
3
4
import torch
5
+import random
6
7
from vllm.sampling_params import SamplingParams, SamplingType
8
from vllm.sequence import SequenceData
9
from vllm.utils import in_wsl, is_neuron
10
+from vllm.model_executor.layers.ops.sample import (
11
+ get_num_triton_sampler_splits)
12
13
_SAMPLING_EPS = 1e-5
14
+_SEED_0_REPLACEMENT = 3403598558
15
16
17
class SamplingMetadata:
@@ -67,14 +71,28 @@ class SamplingTensors:
71
presence_penalties: torch.Tensor
72
frequency_penalties: torch.Tensor
73
repetition_penalties: torch.Tensor
74
+ sampling_seeds: torch.Tensor
75
+ sample_indices: torch.Tensor
76
+ extra_seeds: Optional[torch.Tensor]
77
prompt_tokens: torch.Tensor
78
output_tokens: torch.Tensor
79
80
@classmethod
81
def from_sampling_metadata(
75
- cls, sampling_metadata: "SamplingMetadata", vocab_size: int,
76
- device: torch.device,
77
- dtype: torch.dtype) -> Tuple["SamplingTensors", bool, bool, bool]:
82
+ cls,
83
+ sampling_metadata: "SamplingMetadata",
84
+ vocab_size: int,
85
+ device: torch.device,
86
+ dtype: torch.dtype,
87
+ *,
88
+ extra_seeds_to_generate: int = 0,
89
+ extra_entropy: Optional[Tuple[int, ...]] = None
90
+ ) -> Tuple["SamplingTensors", bool, bool, bool]:
91
+ """
92
+ extra_seeds_to_generate: extra seeds to generate using the
93
+ user-defined seed for each sequence.
94
+ extra_entropy: extra entropy to use when generating seeds.
95
+ """
96
prompt_tokens: List[List[int]] = []
97
output_tokens: List[List[int]] = []
98
top_ks: List[int] = []
@@ -84,9 +102,18 @@ class SamplingTensors:
102
presence_penalties: List[float] = []
103
frequency_penalties: List[float] = []
104
repetition_penalties: List[float] = []
105
+ sampling_seeds: List[int] = []
106
+ sample_indices: List[int] = []
107
+ prompt_best_of: List[int] = []
108
do_penalties = False
109
do_top_p_top_k = False
110
do_min_p = False
111
+
112
+ # We need one base seed per Triton slice.
113
+ seeds_to_generate = (extra_seeds_to_generate +
114
+ get_num_triton_sampler_splits(vocab_size))
115
+
116
+ sample_indices_start_idx = 0
117
for i, seq_group in enumerate(sampling_metadata.seq_groups):
118
seq_ids, sampling_params = seq_group
119
temperature = sampling_params.temperature
@@ -95,6 +122,10 @@ class SamplingTensors:
122
r = sampling_params.repetition_penalty
123
top_p = sampling_params.top_p
124
min_p = sampling_params.min_p
125
+ seed = sampling_params.seed
126
+
127
+ is_greedy = sampling_params.sampling_type == SamplingType.GREEDY
128
+
129
# k should not be greater than the vocab size.
130
top_k = min(sampling_params.top_k, vocab_size)
131
top_k = vocab_size if top_k == -1 else top_k
@@ -112,6 +143,7 @@ class SamplingTensors:
143
or abs(f) >= _SAMPLING_EPS
144
or abs(r - 1.0) >= _SAMPLING_EPS):
145
do_penalties = True
146
+
147
if (i < sampling_metadata.num_prompts
148
and sampling_params.prompt_logprobs is not None):
149
# For tokens in the prompt that we only need to get
@@ -138,10 +170,34 @@ class SamplingTensors:
170
frequency_penalties += [f] * len(seq_ids)
171
repetition_penalties += [r] * len(seq_ids)
172
173
+ is_prompt = i < sampling_metadata.num_prompts
174
+ if is_prompt:
175
+ prompt_best_of.append(sampling_params.best_of)
176
+ prompt_len = sampling_metadata.prompt_lens[i]
177
+
178
+ if sampling_params.prompt_logprobs is not None:
179
+ # NOTE: the sampling position is the last token
180
+ # in the prompt
181
+ sample_indices_start_idx += prompt_len - 1
182
+ for seq_id in seq_ids:
183
+ seq_data = sampling_metadata.seq_data[seq_id]
184
+ extra_entropy = extra_entropy or ()
185
+ seq_seeds = cls._get_sequence_seeds(
186
+ seed,
187
+ seq_data.get_len(),
188
+ *extra_entropy,
189
+ seq_id,
190
+ seeds_to_generate=seeds_to_generate,
191
+ is_greedy=is_greedy)
192
+ sampling_seeds.append(seq_seeds)
193
+ sample_indices.append(sample_indices_start_idx)
194
+ sample_indices_start_idx += 1
195
+
196
sampling_tensors = SamplingTensors.from_lists(
197
temperatures, top_ps, top_ks, min_ps, presence_penalties,
143
- frequency_penalties, repetition_penalties, prompt_tokens,
144
- output_tokens, vocab_size, device, dtype)
198
+ frequency_penalties, repetition_penalties, sampling_seeds,
199
+ sample_indices, prompt_tokens, output_tokens, vocab_size,
200
+ extra_seeds_to_generate, device, dtype)
201
return (sampling_tensors, do_penalties, do_top_p_top_k, do_min_p)
202
203
@classmethod
@@ -150,9 +206,10 @@ class SamplingTensors:
206
presence_penalties: List[float],
207
frequency_penalties: List[float],
208
repetition_penalties: List[float],
209
+ sampling_seeds: List[int], sample_indices: List[int],
210
prompt_tokens: List[List[int]],
211
output_tokens: List[List[int]], vocab_size: int,
155
- device: torch.device,
212
+ extra_seeds_to_generate: int, device: torch.device,
213
dtype: torch.dtype) -> "SamplingTensors":
214
# Note that the performance will be very bad without
215
# pinned memory.
@@ -210,6 +267,12 @@ class SamplingTensors:
267
dtype=torch.int,
268
pin_memory=pin_memory,
269
)
270
+ sample_indices_t = torch.tensor(
271
+ sample_indices,
272
+ device="cpu",
273
+ dtype=torch.long,
274
+ pin_memory=pin_memory,
275
+ )
276
prompt_tensor = torch.tensor(
277
prompt_padded_tokens,
278
device="cpu",
@@ -222,8 +285,28 @@ class SamplingTensors:
285
dtype=torch.long,
286
pin_memory=pin_memory,
287
)
288
+ # need to transpose and make contiguous to
289
+ # copy the tensor correctly.
290
+ # [batch_size, n_seeds] -> [n_seeds, batch_size]
291
+ sampling_seeds_t = torch.tensor(
292
+ sampling_seeds,
293
+ device="cpu",
294
+ dtype=torch.long,
295
+ pin_memory=pin_memory,
296
+ ).T.contiguous()
297
+
298
# Because the memory is pinned, we can do non-blocking
299
# transfer to device.
300
+
301
+ # How many seeds the sample operation itself will need.
302
+ num_base_seeds = sampling_seeds_t.shape[0] - extra_seeds_to_generate
303
+ sampling_seeds_gpu = sampling_seeds_t.to(device=device,
304
+ non_blocking=True)
305
+ extra_seeds_gpu = sampling_seeds_gpu[num_base_seeds:]
306
+ if not extra_seeds_gpu.numel():
307
+ extra_seeds_gpu = None
308
+ sampling_seeds_gpu = sampling_seeds_gpu[:num_base_seeds]
309
+
310
return cls(
311
temperatures=temperatures_t.to(device=device, non_blocking=True),
312
top_ps=top_ps_t.to(device=device, non_blocking=True),
@@ -237,4 +320,38 @@ class SamplingTensors:
320
non_blocking=True),
321
prompt_tokens=prompt_tensor.to(device=device, non_blocking=True),
322
output_tokens=output_tensor.to(device=device, non_blocking=True),
323
+ sampling_seeds=sampling_seeds_gpu,
324
+ sample_indices=sample_indices_t.to(device=device,
325
+ non_blocking=True),
326
+ extra_seeds=extra_seeds_gpu,
327
)
328
+
329
+ @staticmethod
330
+ def _get_sequence_seeds(
331
+ seed: int,
332
+ *extra_entropy: int,
333
+ seeds_to_generate: int,
334
+ is_greedy: bool,
335
+ ):
336
+ """Get `seeds_to_generate` child seeds from `seed` and extra entropy."""
337
+ if not is_greedy:
338
+ if seed is None:
339
+ randint_fn = random.randint
340
+ else:
341
+ generator = random.Random(str((seed, ) + extra_entropy))
342
+ randint_fn = generator.randint
343
+ lo, hi = torch.iinfo(torch.long).min, torch.iinfo(torch.long).max
344
+ # If the user/random sets seed = 0 but request should
345
+ # have sampling, we need to change it to something
346
+ # else. We use a constant in that case.
347
+ # This way we don't need to create and load a bool
348
+ # matrix in the sampling kernel, which reduces CPU
349
+ # overhead and latency.
350
+ seq_seeds = [
351
+ randint_fn(lo, hi) or _SEED_0_REPLACEMENT
352
+ for _ in range(seeds_to_generate)
353
+ ]
354
+ else:
355
+ # For the kernel, seed == 0 means greedy decoding.
356
+ seq_seeds = [0] * seeds_to_generate
357
+ return seq_seeds
vllm/sequence.py
+3
@@ -242,6 +242,9 @@ class Sequence:
242
def get_token_ids(self) -> List[int]:
243
return self.data.get_token_ids()
244
245
+ def get_prompt_token_ids(self) -> List[int]:
246
+ return self.data.get_prompt_token_ids()
247
+
248
def get_last_token_id(self) -> int:
249
return self.data.get_last_token_id()
250
vllm/worker/model_runner.py
+31
-9
@@ -408,6 +408,7 @@ class ModelRunner:
408
selected_token_start_idx = 0
409
categorized_sample_indices = {t: [] for t in SamplingType}
410
categorized_sample_indices_start_idx = 0
411
+ categorized_sampled_token_indices_start_idx = 0
412
pin_memory = not self.in_wsl and not self.device_config.is_neuron
413
414
max_subquery_len = max(subquery_lens) if subquery_lens else 1
@@ -425,9 +426,12 @@ class ModelRunner:
426
categorized_sample_indices_start_idx += subquery_len - 1
427
428
categorized_sample_indices[
428
- sampling_params.sampling_type].append(
429
- categorized_sample_indices_start_idx)
429
+ sampling_params.sampling_type].append([
430
+ categorized_sample_indices_start_idx,
431
+ categorized_sampled_token_indices_start_idx
432
+ ])
433
categorized_sample_indices_start_idx += 1
434
+ categorized_sampled_token_indices_start_idx += 1
435
436
if sampling_params.prompt_logprobs is not None:
437
selected_token_indices.extend(
@@ -449,9 +453,17 @@ class ModelRunner:
453
454
categorized_sample_indices[
455
sampling_params.sampling_type].extend(
452
- range(categorized_sample_indices_start_idx,
453
- categorized_sample_indices_start_idx + num_seqs))
456
+ zip(
457
+ range(
458
+ categorized_sample_indices_start_idx,
459
+ categorized_sample_indices_start_idx +
460
+ num_seqs),
461
+ range(
462
+ categorized_sampled_token_indices_start_idx,
463
+ categorized_sampled_token_indices_start_idx +
464
+ num_seqs)))
465
categorized_sample_indices_start_idx += num_seqs
466
+ categorized_sampled_token_indices_start_idx += num_seqs
467
468
if sampling_params.seed is not None:
469
generators.append(seq_group_metadata.state.generator)
@@ -459,12 +471,14 @@ class ModelRunner:
471
selected_token_indices = _async_h2d(selected_token_indices,
472
dtype=torch.long,
473
target_device=self.device,
462
- pin_memory=pin_memory)
474
+ pin_memory=not self.in_wsl)
475
+
476
categorized_sample_indices = {
464
- t: _async_h2d(seq_ids,
465
- dtype=torch.int,
466
- target_device=self.device,
467
- pin_memory=pin_memory)
477
+ t: _maybe_expand_dim(
478
+ _async_h2d(seq_ids,
479
+ dtype=torch.int,
480
+ target_device=self.device,
481
+ pin_memory=pin_memory), 2, 2)
482
for t, seq_ids in categorized_sample_indices.items()
483
}
484
@@ -884,3 +898,11 @@ def _async_h2d(
898
) -> torch.Tensor:
899
t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
900
return t.to(device=target_device, non_blocking=True)
901
+
902
+
903
+def _maybe_expand_dim(tensor: torch.Tensor,
904
+ target_dims: int,
905
+ size: int = 1) -> torch.Tensor:
906
+ if tensor.ndim < target_dims:
907
+ tensor = tensor.view(-1, *([size] * (target_dims - tensor.ndim)))
908
+ return tensor