@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 426ec4ec67
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[1/n] Triton sampling kernel (#3186)

Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>

Antoni Baum committed Mar 20, 2024 at 14:45 UTC 426ec4ec6711b4180538cd56b9f6b856e5276a1f
10 files changed +1072 -24
tests/kernels/test_rand.py new
+51
@@ -0,0 +1,51 @@
1 +import torch
2 +import pytest
3 +import random
4 +
5 +from vllm.model_executor.layers.ops.rand import seeded_uniform
6 +from vllm.model_executor.utils import set_random_seed
7 +
8 +
9 +@pytest.mark.parametrize("dtype",
10 + [torch.float32, torch.float16, torch.bfloat16])
11 +@pytest.mark.parametrize("use_3d", [True, False])
12 +def test_seeded_uniform(dtype: torch.dtype, use_3d: bool):
13 + device = "cuda"
14 + for seed in range(512):
15 + set_random_seed(seed)
16 + rows = random.randint(1, 512)
17 + cols = random.randint(1, 64000)
18 + if use_3d:
19 + third_dim = random.randint(2, 10)
20 + dims = [rows, third_dim, cols]
21 + else:
22 + dims = [rows, cols]
23 + seeds = torch.randint(torch.iinfo(torch.long).min,
24 + torch.iinfo(torch.long).max, (rows, ),
25 + device=device)
26 +
27 + # Test that the same seed produces the same output
28 + out = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
29 + out2 = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
30 + torch.testing.assert_close(out, out2)
31 + # del to save memory
32 + del out2
33 +
34 + out3 = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
35 + torch.testing.assert_close(out, out3)
36 + # del to save memory
37 + del out3
38 +
39 + # Initialize out tensor with garbage to ensure that it is overwritten
40 + out_with_tensor = seeded_uniform(
41 + *dims,
42 + out=torch.full(
43 + (*dims, ),
44 + -1,
45 + dtype=dtype,
46 + device=device,
47 + ),
48 + seeds=seeds,
49 + dtype=dtype,
50 + )
51 + torch.testing.assert_close(out, out_with_tensor)
tests/kernels/test_sampler.py new
+196
@@ -0,0 +1,196 @@
1 +import gc
2 +
3 +import torch
4 +import pytest
5 +import triton
6 +import triton.language as tl
7 +
8 +from vllm.model_executor.layers.ops.sample import (
9 + _uniform_to_exponential, sample, get_num_triton_sampler_splits,
10 + MAX_TRITON_N_COLS)
11 +from vllm.model_executor.utils import set_random_seed
12 +from vllm.model_executor.sampling_metadata import SamplingTensors
13 +
14 +SINGLE_SPLIT_VOCAB_SIZE = 32000 # llama/mistral/mixtral vocab size
15 +MULTI_SPLIT_VOCAB_SIZE = MAX_TRITON_N_COLS + 100
16 +
17 +
18 +@pytest.fixture(autouse=True)
19 +def _cleanup():
20 + yield
21 + gc.collect()
22 + torch.cuda.empty_cache()
23 +
24 +
25 +@triton.jit
26 +def _uniform_to_exponential_kernel(input, output, n: tl.constexpr):
27 + idx = tl.arange(0, n)
28 + x = tl.load(input + idx)
29 + y = _uniform_to_exponential(x)
30 + tl.store(output + idx, y)
31 +
32 +
33 +def test_uniform_to_exponential():
34 + """Test that we can convert uniform to exponential without div by 0."""
35 + input = torch.tensor([0.0, 1.0 - torch.finfo(torch.float32).eps],
36 + dtype=torch.float32,
37 + device="cuda")
38 + output = torch.zeros(input.shape, dtype=torch.float32, device="cuda")
39 + _uniform_to_exponential_kernel[(1, )](input, output, 2)
40 + assert torch.all(torch.isfinite(output))
41 + assert torch.all(output > 0)
42 + assert torch.all(torch.isfinite(torch.full_like(output, 1.0) / output))
43 +
44 +
45 +@pytest.mark.parametrize("random_sampling", [True, False, "mixed"])
46 +@pytest.mark.parametrize("max_best_of", [1, 2, 3, 4, 5])
47 +@pytest.mark.parametrize("modify_greedy_probs", [True, False])
48 +@pytest.mark.parametrize("seed", [1337])
49 +@pytest.mark.parametrize("vocab_size",
50 + [SINGLE_SPLIT_VOCAB_SIZE, MULTI_SPLIT_VOCAB_SIZE])
51 +@pytest.mark.parametrize("save_logprobs", [True, False])
52 +def test_sample_decoding_only(random_sampling, max_best_of,
53 + modify_greedy_probs, seed, vocab_size,
54 + save_logprobs):
55 + set_random_seed(seed)
56 + bs = 8
57 + probs = torch.zeros((bs, vocab_size), dtype=torch.float32, device="cuda")
58 + for i in range(bs):
59 + probs[i, i * (vocab_size // bs)] = 1.0
60 + logprobs = torch.rand_like(probs)
61 + sample_indices = torch.arange(bs, dtype=torch.long, device="cuda")
62 + n_splits = get_num_triton_sampler_splits(probs.shape[1])
63 + if random_sampling == "mixed":
64 + random_sampling_mask = (torch.rand(
65 + (1, bs), device="cuda") < 0.5).expand(n_splits, bs)
66 + elif random_sampling:
67 + random_sampling_mask = torch.ones((n_splits, bs),
68 + dtype=torch.bool,
69 + device="cuda")
70 + else:
71 + random_sampling_mask = torch.zeros((n_splits, bs),
72 + dtype=torch.bool,
73 + device="cuda")
74 +
75 + seeds = torch.randint(1,
76 + torch.iinfo(torch.long).max, (n_splits, bs),
77 + device="cuda").mul_(random_sampling_mask)
78 + sampled_tokens, sampled_logprobs, sampled_modified_probs = sample(
79 + probs=probs,
80 + logprobs=logprobs,
81 + sample_indices=sample_indices,
82 + seeds=seeds,
83 + max_best_of=max_best_of,
84 + modify_greedy_probs=modify_greedy_probs,
85 + save_logprobs=save_logprobs,
86 + _save_modified_probs=True)
87 + assert sampled_tokens.shape == (bs, max_best_of)
88 + for i in range(bs):
89 + assert torch.all(sampled_tokens[i] == i * (vocab_size // bs))
90 + request_uses_random_sampling = random_sampling_mask[0, i]
91 + if modify_greedy_probs and not request_uses_random_sampling:
92 + # If we are modifying greedy probs and the request is greedy,
93 + # we want to make sure the probs tensor is modified in place
94 + assert torch.allclose(
95 + probs[i][sampled_tokens[i]],
96 + torch.full_like(probs[i][sampled_tokens[i]], 1.0))
97 + assert torch.sum(probs[i]) == 1.0
98 + assert torch.allclose(
99 + sampled_modified_probs[i][0],
100 + torch.full_like(sampled_modified_probs[i][0], 1.0))
101 + elif request_uses_random_sampling:
102 + # If the request is random, we want to make sure
103 + # sampled_modified_probs tensor has noise added
104 + # (and thus is different from probs tensor)
105 + assert not torch.allclose(sampled_modified_probs[i][0],
106 + probs[i][sampled_tokens[i]])
107 + elif not request_uses_random_sampling:
108 + # If the request is greedy and we are not modifying greedy probs,
109 + # we want to make sure sampled_modified_probs tensor is the same as
110 + # the probs tensor.
111 + assert torch.allclose(sampled_modified_probs[i][0],
112 + probs[i][sampled_tokens[i]])
113 +
114 + if save_logprobs:
115 + assert sampled_logprobs.shape == (bs, max_best_of)
116 + for i in range(bs):
117 + for best_of in range(max_best_of):
118 + assert torch.all(sampled_logprobs[i] == logprobs[i][
119 + sampled_tokens[i, best_of]])
120 + else:
121 + assert sampled_logprobs is None
122 +
123 +
124 +@pytest.mark.parametrize("random_sampling", [True, False, "mixed"])
125 +@pytest.mark.parametrize("max_best_of", [1, 2, 3, 4, 5])
126 +@pytest.mark.parametrize("modify_greedy_probs", [True, False])
127 +@pytest.mark.parametrize("seed", [1337])
128 +@pytest.mark.parametrize("vocab_size",
129 + [SINGLE_SPLIT_VOCAB_SIZE, MULTI_SPLIT_VOCAB_SIZE])
130 +def test_sample_prompt_logprobs(random_sampling, max_best_of,
131 + modify_greedy_probs, seed, vocab_size):
132 + set_random_seed(seed)
133 + prompt_sizes = [16, 32, 64, 128] * 2
134 + samples = 8
135 + bs = samples + sum(prompt_sizes)
136 + probs = torch.zeros((bs, vocab_size), dtype=torch.float32, device="cuda")
137 + for i in range(bs):
138 + probs[i, i * (vocab_size // bs)] = 1.0
139 + logprobs = torch.rand_like(probs)
140 + sample_indices = torch.tensor(prompt_sizes,
141 + dtype=torch.long,
142 + device="cuda").cumsum_(0)
143 + n_splits = get_num_triton_sampler_splits(probs.shape[1])
144 + if random_sampling == "mixed":
145 + random_sampling_mask = torch.rand(
146 + (n_splits, samples), device="cuda") < 0.5
147 + elif random_sampling:
148 + random_sampling_mask = torch.ones((n_splits, samples),
149 + dtype=torch.bool,
150 + device="cuda")
151 + else:
152 + random_sampling_mask = torch.zeros((n_splits, samples),
153 + dtype=torch.bool,
154 + device="cuda")
155 +
156 + seeds = torch.randint(1,
157 + torch.iinfo(torch.long).max, (n_splits, samples),
158 + device="cuda").mul_(random_sampling_mask)
159 + sampled_tokens, sampled_logprobs, _ = sample(
160 + probs=probs,
161 + logprobs=logprobs,
162 + sample_indices=sample_indices,
163 + seeds=seeds,
164 + max_best_of=max_best_of,
165 + modify_greedy_probs=modify_greedy_probs,
166 + save_logprobs=True)
167 + assert sampled_tokens.shape == (samples, max_best_of)
168 + assert sampled_logprobs.shape == (samples, max_best_of)
169 + for i, t in enumerate(sample_indices):
170 + assert torch.all(sampled_tokens[i] == t * (vocab_size // bs))
171 + for best_of in range(max_best_of):
172 + assert torch.all(sampled_logprobs[i] == logprobs[sample_indices[i]]
173 + [sampled_tokens[i, best_of]])
174 +
175 +
176 +@pytest.mark.parametrize("seed", list(range(16)))
177 +def test_get_sequence_seeds(seed):
178 + """Ensure that we get a different child seed from base
179 + seed + extra entropy"""
180 + starting_seed = seed
181 + seq_seed = None
182 + extra_entropy = 1
183 + for i in range(512):
184 + new_seq_seed = SamplingTensors._get_sequence_seeds(starting_seed,
185 + i,
186 + seeds_to_generate=1,
187 + is_greedy=False)[0]
188 + new_seq_seed_extra_entropy = SamplingTensors._get_sequence_seeds(
189 + starting_seed,
190 + i,
191 + extra_entropy,
192 + seeds_to_generate=1,
193 + is_greedy=False)[0]
194 + assert new_seq_seed_extra_entropy != new_seq_seed
195 + assert seq_seed != new_seq_seed
196 + seq_seed = new_seq_seed
tests/samplers/test_sampler.py
+3 -3
@@ -302,11 +302,11 @@ def test_sampler_logits_processors(seed: int, device: str):
302 batch_size = random.randint(1, 256)
303 input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
304
305 - # This sample logits processor gives infinite score to the i-th token,
305 + # This sample logits processor gives maximum score to the i-th token,
306 # where i is the length of the input sequence.
307 # We therefore expect the output token sequence to be [0, 1, 2, ...]
308 def pick_ith(token_ids, logits):
309 - logits[len(token_ids)] = float("inf")
309 + logits[len(token_ids)] = torch.finfo(logits.dtype).max
310 return logits
311
312 seq_group_metadata_list = []
@@ -385,7 +385,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
385
386 sample_probs = None
387
388 - def mock_sample(probs, logprobs, sampling_metadata):
388 + def mock_sample(probs, *args, **kwargs):
389 nonlocal sample_probs
390 sample_probs = probs
391 return [[prob.topk(1, dim=-1).indices.tolist(), [0]] for prob in probs]
vllm/model_executor/layers/ops/__init__.py
vllm/model_executor/layers/ops/rand.py new
+157
@@ -0,0 +1,157 @@
1 +import torch
2 +import triton
3 +import triton.language as tl
4 +
5 +from typing import Optional, Union
6 +
7 +
8 +def seeded_uniform(
9 + *size,
10 + seeds: torch.Tensor,
11 + out: Optional[torch.Tensor] = None,
12 + dtype: Optional[torch.dtype] = None,
13 + device: Optional[Union[torch.device, str]] = None,
14 + pin_memory: Optional[bool] = False,
15 +) -> torch.Tensor:
16 + """Similar to torch.rand, but allows for seeds to be set per row.
17 +
18 + seeds must be a 1d tensor. The output tensor may be 1d, 2d, or 3d.
19 + If it is 3d, the additional seeds needed will be derived automatically
20 + in a deterministic fashion:
21 + [
22 + row 0: [columns_with_seed_0], [columns_with_seed0^1], ...
23 + ]
24 + """
25 + n_dims = len(size)
26 +
27 + if n_dims > 3:
28 + raise ValueError("seeded_uniform only supports up to 3D tensors")
29 +
30 + if out is None:
31 + out = torch.empty(*size,
32 + dtype=dtype,
33 + device=device,
34 + pin_memory=pin_memory)
35 + elif out.shape != size:
36 + raise ValueError("shape of out and size must be the same")
37 +
38 + if n_dims == 3:
39 + n_rows, n_3d, n_cols = out.shape
40 + stride_row = out.stride(0)
41 + stride_3d = out.stride(1)
42 + elif n_dims == 2:
43 + n_rows, n_cols = out.shape
44 + n_3d = 1
45 + stride_row = out.stride(0)
46 + stride_3d = 1
47 + else:
48 + n_cols = out.shape[0]
49 + n_rows = 1
50 + n_3d = 1
51 + stride_row = 1
52 + stride_3d = 1
53 +
54 + if seeds.ndim != 1:
55 + raise ValueError("seeds must be a 1D tensor")
56 +
57 + if seeds.numel() != n_rows:
58 + raise ValueError(
59 + "seeds must have the same number of elements as out has rows")
60 +
61 + # The philox PRNG Triton uses generates 4 random numbers at once.
62 + # Therefore, the most efficient use of it is to divide the
63 + # block size by 4, and then save the generated random numbers to
64 + # each of the 4 slices of the tensor.
65 + full_block_size = triton.next_power_of_2(n_cols)
66 + philox_block_size = max(full_block_size // 4, 1)
67 + n_slices = full_block_size // philox_block_size
68 + num_warps = 4
69 + # Manual tuning. This seems to give best performance on A100 for
70 + # simple kernels like this.
71 + if philox_block_size >= 8192:
72 + num_warps = 32
73 + elif philox_block_size >= 4096:
74 + num_warps = 16
75 + elif philox_block_size >= 2048:
76 + num_warps = 8
77 +
78 + _seeded_uniform_triton[(n_rows, n_3d)](
79 + out,
80 + seeds,
81 + stride_row,
82 + stride_3d,
83 + seeds.stride(0),
84 + n_rows,
85 + n_3d,
86 + n_cols,
87 + n_slices=n_slices,
88 + num_warps=num_warps,
89 + block_size=philox_block_size,
90 + )
91 + return out
92 +
93 +
94 +@triton.jit
95 +def _seeded_uniform_triton(
96 + out_ptr: torch.Tensor,
97 + seed_ptr: torch.Tensor,
98 + out_row_stride: int,
99 + out_3d_stride: int,
100 + seed_row_stride: int,
101 + n_rows: int,
102 + n_3d: int,
103 + n_cols: int,
104 + n_slices: tl.constexpr,
105 + block_size: tl.constexpr,
106 +):
107 + """
108 + Generate a random float32 number in [0, 1) for each element in the output
109 + tensor. The random numbers in a row generated using the seed for that row.
110 +
111 + Args:
112 + out_ptr: The output tensor.
113 + seed_ptr: The per-row seeds to use for random number generation.
114 + out_row_stride: The stride between rows of the output tensor.
115 + out_3d_stride: The stride between 3D slices of the output tensor.
116 + seed_row_stride: The stride between rows of the seed tensor.
117 + n_rows: The number of rows in the output tensor.
118 + n_3d: The size of second dimension of the output tensor,
119 + if output tensor is 3D.
120 + n_cols: The number of columns in the output tensor.
121 + n_slices: The number of philox outputs to use.
122 + """
123 + tl.static_assert(n_slices > 0 and n_slices <= 4, "0 < n_slices <= 4")
124 +
125 + # Get the row index.
126 + row_idx = tl.program_id(axis=0)
127 + three_d_idx = tl.program_id(axis=1)
128 +
129 + philox_offsets = tl.arange(0, block_size)
130 + # Get the seed for the current element.
131 + seed = tl.load(seed_ptr + row_idx * seed_row_stride)
132 + if three_d_idx > 0:
133 + seed ^= three_d_idx
134 + # Generate random numbers in [0, 1).
135 + out1, out2, out3, out4 = tl.rand4x(seed, philox_offsets)
136 +
137 + output_row_start_ptr = (out_ptr + row_idx * out_row_stride +
138 + three_d_idx * out_3d_stride)
139 + out1_offsets = philox_offsets
140 + tl.store(output_row_start_ptr + out1_offsets,
141 + out1,
142 + mask=out1_offsets < n_cols)
143 + if n_slices > 1:
144 + out2_offsets = tl.arange(block_size, block_size * 2)
145 + tl.store(output_row_start_ptr + out2_offsets,
146 + out2,
147 + mask=out2_offsets < n_cols)
148 + if n_slices > 2:
149 + out3_offsets = tl.arange(block_size * 2, block_size * 3)
150 + tl.store(output_row_start_ptr + out3_offsets,
151 + out3,
152 + mask=out3_offsets < n_cols)
153 + if n_slices > 3:
154 + out4_offsets = tl.arange(block_size * 3, block_size * 4)
155 + tl.store(output_row_start_ptr + out4_offsets,
156 + out4,
157 + mask=out4_offsets < n_cols)
vllm/model_executor/layers/ops/sample.py new
+405
@@ -0,0 +1,405 @@
1 +import math
2 +from typing import Tuple, Optional
3 +
4 +import torch
5 +import triton
6 +import triton.language as tl
7 +
8 +from vllm.model_executor.layers.ops.rand import seeded_uniform
9 +
10 +_EPS = 1e-6
11 +
12 +# This is a hardcoded limit in Triton (max block size).
13 +MAX_TRITON_N_COLS = 131072
14 +
15 +
16 +def get_num_triton_sampler_splits(n_cols: int) -> int:
17 + """Get the number of splits to use for Triton sampling.
18 +
19 + Triton has a limit on the number of columns it can handle, so we need to
20 + split the tensor and call the kernel multiple times if it's too large.
21 + """
22 + return math.ceil(n_cols / MAX_TRITON_N_COLS)
23 +
24 +
25 +def _multi_split_sample(
26 + probs: torch.Tensor,
27 + seeds: torch.Tensor,
28 + n_splits: int,
29 + sampled_tokens_size: Tuple[int, int],
30 + sampled_logprobs_size: Tuple[int, int],
31 + sample_indices: torch.Tensor,
32 + *,
33 + logprobs: Optional[torch.Tensor] = None,
34 + modify_greedy_probs: bool = False,
35 + save_logprobs: bool = False,
36 +):
37 + """Sample tokens where vocab size is split into multiple parts
38 + (too large for Triton otherwise)."""
39 + assert seeds.ndim == 2 and seeds.shape[0] == n_splits
40 + split_probs = probs.tensor_split(n_splits, 1)
41 + split_logprobs = logprobs.tensor_split(n_splits, 1)
42 + sampled_tokens_tmp = [
43 + torch.empty(sampled_tokens_size, dtype=torch.long, device=probs.device)
44 + for _ in range(n_splits)
45 + ]
46 + sampled_logprobs_tmp = [
47 + torch.empty(sampled_logprobs_size,
48 + dtype=probs.dtype,
49 + device=probs.device) for _ in range(n_splits)
50 + ]
51 + # We are purposefuly using sampled_tokens_size as we need to always
52 + # save modified probs in this case.
53 + sampled_modified_probs_tmp = [
54 + torch.empty(sampled_tokens_size,
55 + dtype=probs.dtype,
56 + device=probs.device) for _ in range(n_splits)
57 + ]
58 + for i in range(n_splits):
59 + n_samples = sample_indices.shape[0]
60 + n_cols = split_probs[i].shape[1]
61 + n_best = sampled_tokens_tmp[i].shape[1]
62 + uniform_noise = seeded_uniform(n_samples,
63 + n_best,
64 + n_cols,
65 + seeds=seeds[i].flatten(),
66 + device=split_probs[i].device,
67 + dtype=split_probs[i].dtype)
68 + # TODO(yard1): See if we can remove the contiguous() calls.
69 + # Will need kernel support.
70 + _sample(
71 + split_probs[i].contiguous(),
72 + split_logprobs[i].contiguous(),
73 + sample_indices,
74 + sampled_tokens_tmp[i],
75 + sampled_logprobs_tmp[i],
76 + sampled_modified_probs_tmp[i],
77 + seeds[i],
78 + uniform_noise,
79 + modify_greedy_probs=False,
80 + save_logprobs=save_logprobs,
81 + save_modified_probs=True,
82 + )
83 + if i > 0:
84 + # Add offset to sampled tokens
85 + sampled_tokens_tmp[i].add_(i * split_probs[i - 1].shape[1])
86 + sampled_tokens = torch.stack(sampled_tokens_tmp)
87 + sampled_modified_probs = torch.stack(sampled_modified_probs_tmp)
88 + # Reduce the results from the splits.
89 + sampled_modified_probs, indices = torch.max(sampled_modified_probs,
90 + dim=0,
91 + keepdim=True)
92 + sampled_tokens = sampled_tokens.gather(0, indices).squeeze(0)
93 + if save_logprobs:
94 + sampled_logprobs = torch.stack(sampled_logprobs_tmp)
95 + sampled_logprobs = sampled_logprobs.gather(0, indices).squeeze(0)
96 + else:
97 + sampled_logprobs = None
98 + sampled_modified_probs = sampled_modified_probs.squeeze(0)
99 +
100 + if modify_greedy_probs:
101 + # We need to modify the greedy probs for the sampled tokens.
102 + # We can't do this in the kernel as we need to know the
103 + # sampled tokens.
104 + probs.fill_(0.0)
105 + probs.scatter_(1, sampled_tokens, 1.0)
106 +
107 + return (sampled_tokens, sampled_logprobs, sampled_modified_probs)
108 +
109 +
110 +def sample(
111 + probs: torch.Tensor,
112 + seeds: torch.Tensor,
113 + *,
114 + max_best_of: int = 1,
115 + sample_indices: Optional[torch.Tensor] = None,
116 + logprobs: Optional[torch.Tensor] = None,
117 + modify_greedy_probs: bool = False,
118 + save_logprobs: bool = False,
119 + _save_modified_probs: bool = False, # pylint: disable=invalid-name
120 +) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]:
121 + """Sample tokens from probs. with per-sequence seeds.
122 +
123 + Can sample from a subset of sequences through sample_indices.
124 +
125 + Args:
126 + probs: Probabilities to sample from.
127 + shape = [batch_size, vocab_size]
128 + seeds: Per-sequence seed values.
129 + shape = [n, math.ceil(vocab_size / MAX_TRITON_N_COLS)]
130 + max_best_of: Number of samples to generate per sequence.
131 + Sequence seed will be incremented by 1 each time.
132 + sample_indices: Indices of sequences to sample from.
133 + If not provided, will sample from all sequences.
134 + shape = [n]
135 + logprobs: Log-probabilities of the sampled tokens.
136 + Only used for saving the logprobs if save_logprobs is True.
137 + shape = [batch_size, vocab_size]
138 + modify_greedy_probs: Whether to modify the greedy probabilities
139 + for speculative sampling (sampled token = 1.0,
140 + everything else = 0.0).
141 + save_logprobs: Whether to save the log-probabilities of the
142 + sampled tokens to a tensor.
143 + _save_modified_probs: Whether to save the modified probabilities
144 + (including gumbel noise) of the sampled tokens to a tensor.
145 + DOES NOT include the modification done by modify_greedy_probs
146 + (because we want to use the unmodified probs to pick the best
147 + split in case of multi-split sampling).
148 + This is exposed only for testing.
149 +
150 + Returns:
151 + sampled_tokens: shape = [n, max_best_of]
152 + sampled_logprobs: shape = [n, max_best_of] if save_logprobs else None
153 + sampled_modified_probs: shape = [n, max_best_of]
154 + if save_modified_probs else None
155 + """
156 + if sample_indices is None:
157 + sample_indices = torch.arange(0, probs.shape[0], device=probs.device)
158 +
159 + sampled_tokens_size = (sample_indices.size(0), max_best_of)
160 + if save_logprobs:
161 + if logprobs is None:
162 + raise ValueError(
163 + "logprobs tensor must be provided if save_logprobs is True")
164 + sampled_logprobs_size = sampled_tokens_size
165 + else:
166 + # Empty tensors to invoke the kernel
167 + sampled_logprobs_size = (0, 0)
168 + logprobs = probs
169 +
170 + if _save_modified_probs:
171 + sampled_modified_probs_size = sampled_tokens_size
172 + else:
173 + # Empty tensors to invoke the kernel
174 + sampled_modified_probs_size = (0, 0)
175 +
176 + # If the number of columns in probs is too large for Triton to handle,
177 + # we split the tensor and sample from each split separately, and then
178 + # do an argmax+gather to combine the results.
179 + n_splits = get_num_triton_sampler_splits(probs.shape[1])
180 + if n_splits > 1:
181 + (sampled_tokens, sampled_logprobs,
182 + sampled_modified_probs) = _multi_split_sample(
183 + probs,
184 + seeds,
185 + n_splits,
186 + sampled_tokens_size,
187 + sampled_logprobs_size,
188 + sample_indices,
189 + logprobs=logprobs,
190 + modify_greedy_probs=modify_greedy_probs,
191 + save_logprobs=save_logprobs)
192 + else:
193 + sampled_tokens = torch.empty(sampled_tokens_size,
194 + dtype=torch.long,
195 + device=probs.device)
196 + sampled_logprobs = torch.empty(sampled_logprobs_size,
197 + dtype=probs.dtype,
198 + device=probs.device)
199 + sampled_modified_probs = torch.empty(sampled_modified_probs_size,
200 + dtype=probs.dtype,
201 + device=probs.device)
202 + n_samples = sample_indices.shape[0]
203 + n_cols = probs.shape[1]
204 + uniform_noise = seeded_uniform(n_samples,
205 + max_best_of,
206 + n_cols,
207 + seeds=seeds.flatten(),
208 + device=probs.device,
209 + dtype=probs.dtype)
210 +
211 + _sample(
212 + probs,
213 + logprobs,
214 + sample_indices,
215 + sampled_tokens,
216 + sampled_logprobs,
217 + sampled_modified_probs,
218 + seeds,
219 + uniform_noise,
220 + modify_greedy_probs=modify_greedy_probs,
221 + save_logprobs=save_logprobs,
222 + save_modified_probs=_save_modified_probs,
223 + )
224 + return (sampled_tokens, sampled_logprobs if save_logprobs else None,
225 + sampled_modified_probs if _save_modified_probs else None)
226 +
227 +
228 +def _sample(probs: torch.Tensor,
229 + logprobs: torch.Tensor,
230 + sample_indices: torch.Tensor,
231 + output_samples: torch.Tensor,
232 + output_logprobs: torch.Tensor,
233 + output_modified_probs: torch.Tensor,
234 + seeds: torch.Tensor,
235 + uniform_noise: torch.Tensor,
236 + *,
237 + modify_greedy_probs: bool = False,
238 + save_logprobs: bool = True,
239 + save_modified_probs: bool = False) -> torch.Tensor:
240 + """Sample tokens from probs.
241 +
242 + Args:
243 + probs [batch_size, vocab_size]: probs to sample from.
244 + logprobs [batch_size, vocab_size]: logprobs (used when
245 + save_logprobsis True).
246 + sample_indices [n]: Indices of the samples to use for each row of probs.
247 + output_samples [n, n_best]: Output tensor to store samples in.
248 + output_logprobs [n, n_best]: Output tensor to store logprobs in.
249 + output_modified_probs [n, n_best]: Output tensor to store
250 + probs of chosen tokens in (modified with noise).
251 + seeds [n]: Seeds to use for sampling. If the seed is 0, we use
252 + greedy sampling. Note this is ONLY used for determining
253 + whether to use random sampling or not. The actual random
254 + noise should be passed as uniform_noise.
255 + uniform_noise [batch_size, n_best, vocab_size]: Uniform
256 + noise to use for random sampling (will be converted
257 + to exponential gumbel noise by the kernel).
258 + modify_greedy_probs: If True, we modify the probs tensor in-place
259 + to encode the sampling method used for each row. This is used
260 + in speculative decoding. Only applies in greedy decoding.
261 + save_logprobs: If True, we save the logprobs of the sampled tokens
262 + in the output_logprobs tensor.
263 + save_modified_probs: If True, we save the modified probs (with noise)
264 + of the sampled tokens in the output_modified_probs tensor.
265 + DOES NOT include the modification done by modify_greedy_probs
266 + (because we want to use the unmodified probs to pick the best
267 + split in case of multi-split sampling).
268 + """
269 + n_samples = sample_indices.shape[0]
270 + n_cols = probs.shape[1]
271 + n_best = output_samples.shape[1] if len(output_samples.shape) > 1 else 1
272 +
273 + # The block size is the smallest power of two greater than the number of
274 + # columns in probs
275 + block_size = triton.next_power_of_2(n_cols)
276 + num_warps = 4
277 + # Manual tuning. This seems to give best performance on A100 for
278 + # simple kernels like this.
279 + if block_size >= 8192:
280 + num_warps = 32
281 + elif block_size >= 4096:
282 + num_warps = 16
283 + elif block_size >= 2048:
284 + num_warps = 8
285 +
286 + # Enqueue kernel. The 1D launch grid is simple: we have one kernel
287 + # instance per row of the probs matrix
288 + _sample_triton[(n_samples, n_best)](
289 + sample_indices,
290 + output_samples,
291 + output_logprobs,
292 + output_modified_probs,
293 + probs,
294 + logprobs,
295 + seeds,
296 + uniform_noise,
297 + output_samples.stride(0),
298 + probs.stride(0),
299 + uniform_noise.stride(0),
300 + uniform_noise.stride(1) if n_best > 1 else 1,
301 + n_samples,
302 + n_cols,
303 + n_best,
304 + num_warps=num_warps,
305 + block_size=block_size,
306 + modify_greedy_probs=modify_greedy_probs,
307 + save_logprobs=save_logprobs,
308 + save_modified_probs=save_modified_probs,
309 + )
310 + return output_samples, output_logprobs, output_modified_probs
311 +
312 +
313 +@triton.jit
314 +def _uniform_to_exponential(uniform_noise):
315 + """Convert uniform samples to exponential samples."""
316 + # tl.rand returns values in [0, 1), so we clamp lower bound
317 + # to _EPS to avoid log(0) and thus division by 0 later
318 + lb = tl.full(uniform_noise.shape, _EPS, uniform_noise.dtype)
319 + uniform_noise = tl.maximum(uniform_noise, lb)
320 + # Use the inversion method to turn uniform samples
321 + # into exponential samples
322 + exponential_noise = -tl.log(uniform_noise)
323 + return exponential_noise
324 +
325 +
326 +@triton.jit
327 +def _sample_triton(
328 + sample_indices_ptr: torch.Tensor, output_ptr: torch.Tensor,
329 + output_logprobs_ptr: torch.Tensor,
330 + output_modified_probs_ptr: torch.Tensor, probs_ptr: torch.Tensor,
331 + logprobs_ptr: torch.Tensor, seeds_ptr: torch.Tensor,
332 + uniform_noise_ptr: torch.Tensor, output_row_stride: int,
333 + probs_row_stride: int, uniform_noise_row_stride: int,
334 + uniform_noise_best_stride: int, n_samples: int, n_cols: int,
335 + n_best: int, block_size: tl.constexpr,
336 + modify_greedy_probs: tl.constexpr, save_logprobs: tl.constexpr,
337 + save_modified_probs: tl.constexpr):
338 + # The rows are independent, so we parallelize across those
339 + sample_idx = tl.program_id(0)
340 + best_idx = tl.program_id(1)
341 +
342 + # Load the row index from DRAM
343 + row_idx = tl.load(sample_indices_ptr + sample_idx)
344 + seed = tl.load(seeds_ptr + sample_idx)
345 + uses_random_sampling = seed != 0
346 +
347 + # The stride represents how much we need to increase the
348 + # pointer to advance 1 row
349 + row_start_ptr = probs_ptr + row_idx * probs_row_stride
350 +
351 + # The block size is the next power of two greater than n_cols,
352 + # so we can fit each row in a single block
353 + col_offsets = tl.arange(0, block_size)
354 +
355 + # Load the row into SRAM, using a mask since block_size may be > than n_cols
356 + row = tl.load(row_start_ptr + col_offsets,
357 + mask=col_offsets < n_cols,
358 + other=float("-inf"))
359 +
360 + if uses_random_sampling:
361 + uniform_noise_start_ptr = (uniform_noise_ptr +
362 + sample_idx * uniform_noise_row_stride +
363 + best_idx * uniform_noise_best_stride)
364 + uniform_noise = tl.load(uniform_noise_start_ptr + col_offsets,
365 + mask=col_offsets < n_cols,
366 + other=0.5)
367 + exponential_noise = _uniform_to_exponential(uniform_noise)
368 + row /= exponential_noise
369 +
370 + sampled_value, sampled_token = tl.max(row, axis=0, return_indices=True)
371 + # clamp sampled token to n_cols - 1
372 + # this should not be necessary, but we do it
373 + # just in case
374 + if sampled_token >= n_cols:
375 + sampled_token = n_cols - 1
376 + # Write back output to DRAM
377 + output_row_start_ptr = (output_ptr + sample_idx * output_row_stride +
378 + best_idx)
379 + tl.store(output_row_start_ptr, sampled_token)
380 +
381 + if modify_greedy_probs: # noqa
382 + if not uses_random_sampling:
383 + # Set the probability of the sampled token to 1, all other
384 + # tokens to zero. This is used in speculative decoding where
385 + # the sampling method must be encoded within the sampled
386 + # probability distributions.
387 + row = tl.where(col_offsets == sampled_token, 1.0, 0.0)
388 + tl.store(row_start_ptr + col_offsets,
389 + row,
390 + mask=col_offsets < n_cols)
391 +
392 + if save_modified_probs:
393 + output_row_start_ptr = (output_modified_probs_ptr +
394 + sample_idx * output_row_stride + best_idx)
395 + tl.store(output_row_start_ptr, sampled_value)
396 +
397 + if save_logprobs:
398 + # Load the row into SRAM, using a mask since block_size
399 + # may be > than n_cols
400 + sampled_logprob = tl.load(logprobs_ptr + row_idx * probs_row_stride +
401 + sampled_token)
402 + # Write back output to DRAM
403 + output_row_start_ptr = (output_logprobs_ptr +
404 + sample_idx * output_row_stride + best_idx)
405 + tl.store(output_row_start_ptr, sampled_logprob)
vllm/model_executor/layers/sampler.py
+103 -6
@@ -12,6 +12,7 @@ from vllm.sampling_params import SamplingParams, SamplingType
12 from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
13 SamplerOutput, SequenceData, SequenceGroupOutput,
14 SequenceOutput)
15 +from vllm.model_executor.layers.ops.sample import (sample as sample_triton)
16 from vllm.utils import is_neuron
17
18
@@ -114,7 +115,8 @@ class Sampler(nn.Module):
115 logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
116
117 # Sample the next tokens.
117 - sample_results = _sample(probs, logprobs, sampling_metadata)
118 + sample_results = _sample(probs, logprobs, sampling_metadata,
119 + sampling_tensors)
120 # Get the logprobs query results.
121 prompt_logprobs, sample_logprobs = _get_logprobs(
122 logprobs, sampling_metadata, sample_results)
@@ -375,7 +377,7 @@ def _multinomial(
377 return probs.div_(q).argmax(dim=1).view(-1, num_samples)
378
379
378 -def _sample(
380 +def _sample_with_torch(
381 probs: torch.Tensor,
382 logprobs: torch.Tensor,
383 sampling_metadata: SamplingMetadata,
@@ -394,7 +396,7 @@ def _sample(
396 # Counterintiutively, having two loops here is actually faster.
397 # The first loop can run without waiting on GPU<->CPU sync.
398 for sampling_type in SamplingType:
397 - sample_indices = categorized_sample_indices[sampling_type]
399 + sample_indices = categorized_sample_indices[sampling_type][:, 0]
400 num_tokens = len(sample_indices)
401 if num_tokens == 0:
402 continue
@@ -407,17 +409,19 @@ def _sample(
409 greedy_samples = torch.argmax(logprobs[sample_indices.long()],
410 dim=-1)
411 elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
410 - max_best_of = 1
412 + max_best_of_in_batch = 1
413 for seq_group, is_prompt in zip(seq_groups, is_prompts):
414 if is_prompt:
415 _, sampling_params = seq_group
414 - max_best_of = max(max_best_of, sampling_params.best_of)
416 + max_best_of_in_batch = max(max_best_of_in_batch,
417 + sampling_params.best_of)
418 seeded_args = {} if sampling_type == SamplingType.RANDOM else {
419 "seq_groups": seq_groups,
420 "generators": sampling_metadata.generators,
421 }
422 multinomial_samples[sampling_type] = _multinomial(
420 - probs[sample_indices.long()], max_best_of, **seeded_args)
423 + probs[sample_indices.long()], max_best_of_in_batch,
424 + **seeded_args)
425 elif sampling_type == SamplingType.BEAM:
426 beam_search_logprobs = logprobs[sample_indices]
427 else:
@@ -448,6 +452,99 @@ def _sample(
452 return sample_results
453
454
455 +def _sample_with_triton_kernel(
456 + probs: torch.Tensor,
457 + logprobs: torch.Tensor,
458 + sampling_metadata: SamplingMetadata,
459 + sampling_tensors: SamplingTensors,
460 +) -> List[Tuple[List[int], List[int]]]:
461 + categorized_seq_group_ids = {t: [] for t in SamplingType}
462 + categorized_sample_indices = sampling_metadata.categorized_sample_indices
463 + for i, seq_group in enumerate(sampling_metadata.seq_groups):
464 + _, sampling_params = seq_group
465 + sampling_type = sampling_params.sampling_type
466 + categorized_seq_group_ids[sampling_type].append(i)
467 +
468 + sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
469 + sample_metadata = {}
470 + max_best_of_in_batch = 1
471 +
472 + # Counterintiutively, having two loops here is actually faster.
473 + # The first loop can run without waiting on GPU<->CPU sync.
474 + for sampling_type in SamplingType:
475 + sample_indices = categorized_sample_indices[sampling_type][:, 0]
476 + sampled_token_indices = categorized_sample_indices[sampling_type][:, 1]
477 + num_tokens = len(sample_indices)
478 + if num_tokens == 0:
479 + continue
480 + seq_group_ids = categorized_seq_group_ids[sampling_type]
481 + seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
482 + is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
483 + sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
484 + is_prompts, sample_indices,
485 + sampled_token_indices)
486 + if sampling_type in (SamplingType.GREEDY, SamplingType.RANDOM,
487 + SamplingType.RANDOM_SEED):
488 + for seq_group, is_prompt in zip(seq_groups, is_prompts):
489 + if is_prompt:
490 + _, sampling_params = seq_group
491 + max_best_of_in_batch = max(max_best_of_in_batch,
492 + sampling_params.best_of)
493 + elif sampling_type == SamplingType.BEAM:
494 + beam_search_logprobs = logprobs[sample_indices]
495 + else:
496 + raise ValueError(f"Unsupported sampling type: {sampling_type}")
497 +
498 + sampled_tokens, _, _ = sample_triton(
499 + probs=probs,
500 + seeds=sampling_tensors.sampling_seeds,
501 + max_best_of=max_best_of_in_batch,
502 + sample_indices=sampling_tensors.sample_indices,
503 + logprobs=logprobs,
504 + # don't save logprobs because we have logic for that below
505 + # TODO: use this instead of the CPU-based logic below
506 + save_logprobs=False,
507 + )
508 +
509 + # GPU<->CPU sync happens in the loop below.
510 +
511 + for sampling_type in SamplingType:
512 + if sampling_type not in sample_metadata:
513 + continue
514 + (seq_group_ids, seq_groups, is_prompts, sample_indices,
515 + sampled_token_indices) = sample_metadata[sampling_type]
516 + if sampling_type == SamplingType.GREEDY:
517 + sample_results = _greedy_sample(
518 + seq_groups, sampled_tokens[sampled_token_indices][:, 0])
519 + elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
520 + sample_results = _random_sample(
521 + seq_groups, is_prompts, sampled_tokens[sampled_token_indices])
522 + elif sampling_type == SamplingType.BEAM:
523 + sample_results = _beam_search_sample(seq_groups, is_prompts,
524 + sampling_metadata.seq_data,
525 + beam_search_logprobs)
526 + sample_results_dict.update(zip(seq_group_ids, sample_results))
527 +
528 + sample_results = [
529 + sample_results_dict[i]
530 + for i in range(len(sampling_metadata.seq_groups))
531 + ]
532 + return sample_results
533 +
534 +
535 +def _sample(
536 + probs: torch.Tensor,
537 + logprobs: torch.Tensor,
538 + sampling_metadata: SamplingMetadata,
539 + sampling_tensors: SamplingTensors,
540 +) -> List[Tuple[List[int], List[int]]]:
541 + return _sample_with_torch(probs, logprobs, sampling_metadata)
542 +
543 + # TODO: Enable once Triton kernel & associated code is faster.
544 + # return _sample_with_triton_kernel(probs, logprobs, sampling_metadata,
545 + # sampling_tensors)
546 +
547 +
548 def _get_logprobs(
549 logprobs: torch.Tensor,
550 sampling_metadata: SamplingMetadata,
vllm/model_executor/sampling_metadata.py
+123 -6
@@ -2,12 +2,16 @@ from dataclasses import dataclass
2 from typing import Dict, List, Optional, Tuple
3
4 import torch
5 +import random
6
7 from vllm.sampling_params import SamplingParams, SamplingType
8 from vllm.sequence import SequenceData
9 from vllm.utils import in_wsl, is_neuron
10 +from vllm.model_executor.layers.ops.sample import (
11 + get_num_triton_sampler_splits)
12
13 _SAMPLING_EPS = 1e-5
14 +_SEED_0_REPLACEMENT = 3403598558
15
16
17 class SamplingMetadata:
@@ -67,14 +71,28 @@ class SamplingTensors:
71 presence_penalties: torch.Tensor
72 frequency_penalties: torch.Tensor
73 repetition_penalties: torch.Tensor
74 + sampling_seeds: torch.Tensor
75 + sample_indices: torch.Tensor
76 + extra_seeds: Optional[torch.Tensor]
77 prompt_tokens: torch.Tensor
78 output_tokens: torch.Tensor
79
80 @classmethod
81 def from_sampling_metadata(
75 - cls, sampling_metadata: "SamplingMetadata", vocab_size: int,
76 - device: torch.device,
77 - dtype: torch.dtype) -> Tuple["SamplingTensors", bool, bool, bool]:
82 + cls,
83 + sampling_metadata: "SamplingMetadata",
84 + vocab_size: int,
85 + device: torch.device,
86 + dtype: torch.dtype,
87 + *,
88 + extra_seeds_to_generate: int = 0,
89 + extra_entropy: Optional[Tuple[int, ...]] = None
90 + ) -> Tuple["SamplingTensors", bool, bool, bool]:
91 + """
92 + extra_seeds_to_generate: extra seeds to generate using the
93 + user-defined seed for each sequence.
94 + extra_entropy: extra entropy to use when generating seeds.
95 + """
96 prompt_tokens: List[List[int]] = []
97 output_tokens: List[List[int]] = []
98 top_ks: List[int] = []
@@ -84,9 +102,18 @@ class SamplingTensors:
102 presence_penalties: List[float] = []
103 frequency_penalties: List[float] = []
104 repetition_penalties: List[float] = []
105 + sampling_seeds: List[int] = []
106 + sample_indices: List[int] = []
107 + prompt_best_of: List[int] = []
108 do_penalties = False
109 do_top_p_top_k = False
110 do_min_p = False
111 +
112 + # We need one base seed per Triton slice.
113 + seeds_to_generate = (extra_seeds_to_generate +
114 + get_num_triton_sampler_splits(vocab_size))
115 +
116 + sample_indices_start_idx = 0
117 for i, seq_group in enumerate(sampling_metadata.seq_groups):
118 seq_ids, sampling_params = seq_group
119 temperature = sampling_params.temperature
@@ -95,6 +122,10 @@ class SamplingTensors:
122 r = sampling_params.repetition_penalty
123 top_p = sampling_params.top_p
124 min_p = sampling_params.min_p
125 + seed = sampling_params.seed
126 +
127 + is_greedy = sampling_params.sampling_type == SamplingType.GREEDY
128 +
129 # k should not be greater than the vocab size.
130 top_k = min(sampling_params.top_k, vocab_size)
131 top_k = vocab_size if top_k == -1 else top_k
@@ -112,6 +143,7 @@ class SamplingTensors:
143 or abs(f) >= _SAMPLING_EPS
144 or abs(r - 1.0) >= _SAMPLING_EPS):
145 do_penalties = True
146 +
147 if (i < sampling_metadata.num_prompts
148 and sampling_params.prompt_logprobs is not None):
149 # For tokens in the prompt that we only need to get
@@ -138,10 +170,34 @@ class SamplingTensors:
170 frequency_penalties += [f] * len(seq_ids)
171 repetition_penalties += [r] * len(seq_ids)
172
173 + is_prompt = i < sampling_metadata.num_prompts
174 + if is_prompt:
175 + prompt_best_of.append(sampling_params.best_of)
176 + prompt_len = sampling_metadata.prompt_lens[i]
177 +
178 + if sampling_params.prompt_logprobs is not None:
179 + # NOTE: the sampling position is the last token
180 + # in the prompt
181 + sample_indices_start_idx += prompt_len - 1
182 + for seq_id in seq_ids:
183 + seq_data = sampling_metadata.seq_data[seq_id]
184 + extra_entropy = extra_entropy or ()
185 + seq_seeds = cls._get_sequence_seeds(
186 + seed,
187 + seq_data.get_len(),
188 + *extra_entropy,
189 + seq_id,
190 + seeds_to_generate=seeds_to_generate,
191 + is_greedy=is_greedy)
192 + sampling_seeds.append(seq_seeds)
193 + sample_indices.append(sample_indices_start_idx)
194 + sample_indices_start_idx += 1
195 +
196 sampling_tensors = SamplingTensors.from_lists(
197 temperatures, top_ps, top_ks, min_ps, presence_penalties,
143 - frequency_penalties, repetition_penalties, prompt_tokens,
144 - output_tokens, vocab_size, device, dtype)
198 + frequency_penalties, repetition_penalties, sampling_seeds,
199 + sample_indices, prompt_tokens, output_tokens, vocab_size,
200 + extra_seeds_to_generate, device, dtype)
201 return (sampling_tensors, do_penalties, do_top_p_top_k, do_min_p)
202
203 @classmethod
@@ -150,9 +206,10 @@ class SamplingTensors:
206 presence_penalties: List[float],
207 frequency_penalties: List[float],
208 repetition_penalties: List[float],
209 + sampling_seeds: List[int], sample_indices: List[int],
210 prompt_tokens: List[List[int]],
211 output_tokens: List[List[int]], vocab_size: int,
155 - device: torch.device,
212 + extra_seeds_to_generate: int, device: torch.device,
213 dtype: torch.dtype) -> "SamplingTensors":
214 # Note that the performance will be very bad without
215 # pinned memory.
@@ -210,6 +267,12 @@ class SamplingTensors:
267 dtype=torch.int,
268 pin_memory=pin_memory,
269 )
270 + sample_indices_t = torch.tensor(
271 + sample_indices,
272 + device="cpu",
273 + dtype=torch.long,
274 + pin_memory=pin_memory,
275 + )
276 prompt_tensor = torch.tensor(
277 prompt_padded_tokens,
278 device="cpu",
@@ -222,8 +285,28 @@ class SamplingTensors:
285 dtype=torch.long,
286 pin_memory=pin_memory,
287 )
288 + # need to transpose and make contiguous to
289 + # copy the tensor correctly.
290 + # [batch_size, n_seeds] -> [n_seeds, batch_size]
291 + sampling_seeds_t = torch.tensor(
292 + sampling_seeds,
293 + device="cpu",
294 + dtype=torch.long,
295 + pin_memory=pin_memory,
296 + ).T.contiguous()
297 +
298 # Because the memory is pinned, we can do non-blocking
299 # transfer to device.
300 +
301 + # How many seeds the sample operation itself will need.
302 + num_base_seeds = sampling_seeds_t.shape[0] - extra_seeds_to_generate
303 + sampling_seeds_gpu = sampling_seeds_t.to(device=device,
304 + non_blocking=True)
305 + extra_seeds_gpu = sampling_seeds_gpu[num_base_seeds:]
306 + if not extra_seeds_gpu.numel():
307 + extra_seeds_gpu = None
308 + sampling_seeds_gpu = sampling_seeds_gpu[:num_base_seeds]
309 +
310 return cls(
311 temperatures=temperatures_t.to(device=device, non_blocking=True),
312 top_ps=top_ps_t.to(device=device, non_blocking=True),
@@ -237,4 +320,38 @@ class SamplingTensors:
320 non_blocking=True),
321 prompt_tokens=prompt_tensor.to(device=device, non_blocking=True),
322 output_tokens=output_tensor.to(device=device, non_blocking=True),
323 + sampling_seeds=sampling_seeds_gpu,
324 + sample_indices=sample_indices_t.to(device=device,
325 + non_blocking=True),
326 + extra_seeds=extra_seeds_gpu,
327 )
328 +
329 + @staticmethod
330 + def _get_sequence_seeds(
331 + seed: int,
332 + *extra_entropy: int,
333 + seeds_to_generate: int,
334 + is_greedy: bool,
335 + ):
336 + """Get `seeds_to_generate` child seeds from `seed` and extra entropy."""
337 + if not is_greedy:
338 + if seed is None:
339 + randint_fn = random.randint
340 + else:
341 + generator = random.Random(str((seed, ) + extra_entropy))
342 + randint_fn = generator.randint
343 + lo, hi = torch.iinfo(torch.long).min, torch.iinfo(torch.long).max
344 + # If the user/random sets seed = 0 but request should
345 + # have sampling, we need to change it to something
346 + # else. We use a constant in that case.
347 + # This way we don't need to create and load a bool
348 + # matrix in the sampling kernel, which reduces CPU
349 + # overhead and latency.
350 + seq_seeds = [
351 + randint_fn(lo, hi) or _SEED_0_REPLACEMENT
352 + for _ in range(seeds_to_generate)
353 + ]
354 + else:
355 + # For the kernel, seed == 0 means greedy decoding.
356 + seq_seeds = [0] * seeds_to_generate
357 + return seq_seeds
vllm/sequence.py
+3
@@ -242,6 +242,9 @@ class Sequence:
242 def get_token_ids(self) -> List[int]:
243 return self.data.get_token_ids()
244
245 + def get_prompt_token_ids(self) -> List[int]:
246 + return self.data.get_prompt_token_ids()
247 +
248 def get_last_token_id(self) -> int:
249 return self.data.get_last_token_id()
250
vllm/worker/model_runner.py
+31 -9
@@ -408,6 +408,7 @@ class ModelRunner:
408 selected_token_start_idx = 0
409 categorized_sample_indices = {t: [] for t in SamplingType}
410 categorized_sample_indices_start_idx = 0
411 + categorized_sampled_token_indices_start_idx = 0
412 pin_memory = not self.in_wsl and not self.device_config.is_neuron
413
414 max_subquery_len = max(subquery_lens) if subquery_lens else 1
@@ -425,9 +426,12 @@ class ModelRunner:
426 categorized_sample_indices_start_idx += subquery_len - 1
427
428 categorized_sample_indices[
428 - sampling_params.sampling_type].append(
429 - categorized_sample_indices_start_idx)
429 + sampling_params.sampling_type].append([
430 + categorized_sample_indices_start_idx,
431 + categorized_sampled_token_indices_start_idx
432 + ])
433 categorized_sample_indices_start_idx += 1
434 + categorized_sampled_token_indices_start_idx += 1
435
436 if sampling_params.prompt_logprobs is not None:
437 selected_token_indices.extend(
@@ -449,9 +453,17 @@ class ModelRunner:
453
454 categorized_sample_indices[
455 sampling_params.sampling_type].extend(
452 - range(categorized_sample_indices_start_idx,
453 - categorized_sample_indices_start_idx + num_seqs))
456 + zip(
457 + range(
458 + categorized_sample_indices_start_idx,
459 + categorized_sample_indices_start_idx +
460 + num_seqs),
461 + range(
462 + categorized_sampled_token_indices_start_idx,
463 + categorized_sampled_token_indices_start_idx +
464 + num_seqs)))
465 categorized_sample_indices_start_idx += num_seqs
466 + categorized_sampled_token_indices_start_idx += num_seqs
467
468 if sampling_params.seed is not None:
469 generators.append(seq_group_metadata.state.generator)
@@ -459,12 +471,14 @@ class ModelRunner:
471 selected_token_indices = _async_h2d(selected_token_indices,
472 dtype=torch.long,
473 target_device=self.device,
462 - pin_memory=pin_memory)
474 + pin_memory=not self.in_wsl)
475 +
476 categorized_sample_indices = {
464 - t: _async_h2d(seq_ids,
465 - dtype=torch.int,
466 - target_device=self.device,
467 - pin_memory=pin_memory)
477 + t: _maybe_expand_dim(
478 + _async_h2d(seq_ids,
479 + dtype=torch.int,
480 + target_device=self.device,
481 + pin_memory=pin_memory), 2, 2)
482 for t, seq_ids in categorized_sample_indices.items()
483 }
484
@@ -884,3 +898,11 @@ def _async_h2d(
898 ) -> torch.Tensor:
899 t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
900 return t.to(device=target_device, non_blocking=True)
901 +
902 +
903 +def _maybe_expand_dim(tensor: torch.Tensor,
904 + target_dims: int,
905 + size: int = 1) -> torch.Tensor:
906 + if tensor.ndim < target_dims:
907 + tensor = tensor.view(-1, *([size] * (target_dims - tensor.ndim)))
908 + return tensor