@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 546034b466
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[refactor] remove triton based sampler (#8524)

Simon Mo committed Sep 16, 2024 at 20:04 UTC 546034b466bf11f12936791312981b9982850eb0
9 files changed +75 -1095
tests/kernels/test_rand.py deleted
-52
@@ -1,52 +0,0 @@
1 -import random
2 -
3 -import pytest
4 -import torch
5 -
6 -from vllm.model_executor.layers.ops.rand import seeded_uniform
7 -from vllm.model_executor.utils import set_random_seed
8 -
9 -
10 -@pytest.mark.parametrize("dtype",
11 - [torch.float32, torch.float16, torch.bfloat16])
12 -@pytest.mark.parametrize("use_3d", [True, False])
13 -def test_seeded_uniform(dtype: torch.dtype, use_3d: bool):
14 - device = "cuda"
15 - for seed in range(512):
16 - set_random_seed(seed)
17 - rows = random.randint(1, 512)
18 - cols = random.randint(1, 64000)
19 - if use_3d:
20 - third_dim = random.randint(2, 10)
21 - dims = [rows, third_dim, cols]
22 - else:
23 - dims = [rows, cols]
24 - seeds = torch.randint(torch.iinfo(torch.long).min,
25 - torch.iinfo(torch.long).max, (rows, ),
26 - device=device)
27 -
28 - # Test that the same seed produces the same output
29 - out = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
30 - out2 = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
31 - torch.testing.assert_close(out, out2)
32 - # del to save memory
33 - del out2
34 -
35 - out3 = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
36 - torch.testing.assert_close(out, out3)
37 - # del to save memory
38 - del out3
39 -
40 - # Initialize out tensor with garbage to ensure that it is overwritten
41 - out_with_tensor = seeded_uniform(
42 - *dims,
43 - out=torch.full(
44 - (*dims, ),
45 - -1,
46 - dtype=dtype,
47 - device=device,
48 - ),
49 - seeds=seeds,
50 - dtype=dtype,
51 - )
52 - torch.testing.assert_close(out, out_with_tensor)
tests/kernels/test_sampler.py deleted
-209
@@ -1,209 +0,0 @@
1 -import gc
2 -from unittest.mock import patch
3 -
4 -import pytest
5 -import torch
6 -import triton
7 -import triton.language as tl
8 -
9 -from vllm.model_executor.layers.ops.sample import (_sample_triton,
10 - _uniform_to_exponential,
11 - sample)
12 -from vllm.model_executor.sampling_metadata import SamplingTensors
13 -from vllm.model_executor.utils import set_random_seed
14 -from vllm.triton_utils.libentry import LibEntry
15 -from vllm.triton_utils.sample import (MAX_TRITON_N_COLS,
16 - get_num_triton_sampler_splits)
17 -
18 -SINGLE_SPLIT_VOCAB_SIZE = 32000 # llama/mistral/mixtral vocab size
19 -MULTI_SPLIT_VOCAB_SIZE = MAX_TRITON_N_COLS + 100
20 -
21 -
22 -@pytest.fixture(autouse=True)
23 -def _cleanup():
24 - yield
25 - gc.collect()
26 - torch.cuda.empty_cache()
27 -
28 -
29 -@triton.jit
30 -def _uniform_to_exponential_kernel(input, output, n: tl.constexpr):
31 - idx = tl.arange(0, n)
32 - x = tl.load(input + idx)
33 - y = _uniform_to_exponential(x)
34 - tl.store(output + idx, y)
35 -
36 -
37 -def test_uniform_to_exponential():
38 - """Test that we can convert uniform to exponential without div by 0."""
39 - input = torch.tensor([0.0, 1.0 - torch.finfo(torch.float32).eps],
40 - dtype=torch.float32,
41 - device="cuda")
42 - output = torch.zeros(input.shape, dtype=torch.float32, device="cuda")
43 - _uniform_to_exponential_kernel[(1, )](input, output, 2)
44 - assert torch.all(torch.isfinite(output))
45 - assert torch.all(output > 0)
46 - assert torch.all(torch.isfinite(torch.full_like(output, 1.0) / output))
47 -
48 -
49 -@pytest.mark.parametrize("random_sampling", [True, False, "mixed"])
50 -@pytest.mark.parametrize("max_best_of", [1, 2, 3, 4, 5])
51 -@pytest.mark.parametrize("modify_greedy_probs", [True, False])
52 -@pytest.mark.parametrize("seed", [1337])
53 -@pytest.mark.parametrize("vocab_size",
54 - [SINGLE_SPLIT_VOCAB_SIZE, MULTI_SPLIT_VOCAB_SIZE])
55 -@pytest.mark.parametrize("save_logprobs", [True, False])
56 -def test_sample_decoding_only(random_sampling, max_best_of,
57 - modify_greedy_probs, seed, vocab_size,
58 - save_logprobs):
59 - set_random_seed(seed)
60 - bs = 8
61 - probs = torch.zeros((bs, vocab_size), dtype=torch.float32, device="cuda")
62 - for i in range(bs):
63 - probs[i, i * (vocab_size // bs)] = 1.0
64 - logprobs = torch.rand_like(probs)
65 - sample_indices = torch.arange(bs, dtype=torch.long, device="cuda")
66 - n_splits = get_num_triton_sampler_splits(probs.shape[1])
67 - if random_sampling == "mixed":
68 - random_sampling_mask = (torch.rand(
69 - (1, bs), device="cuda") < 0.5).expand(n_splits, bs)
70 - elif random_sampling:
71 - random_sampling_mask = torch.ones((n_splits, bs),
72 - dtype=torch.bool,
73 - device="cuda")
74 - else:
75 - random_sampling_mask = torch.zeros((n_splits, bs),
76 - dtype=torch.bool,
77 - device="cuda")
78 -
79 - seeds = torch.randint(1,
80 - torch.iinfo(torch.long).max, (n_splits, bs),
81 - device="cuda").mul_(random_sampling_mask)
82 - #The current _sample_triton does not utilize the
83 - # libentry decoration. The purpose of adding this patch is to test
84 - # the correctness of libentry.
85 - with patch("vllm.model_executor.layers.ops.sample._sample_triton",
86 - LibEntry(_sample_triton)):
87 - sampled_tokens, sampled_logprobs, sampled_modified_probs = sample(
88 - probs=probs,
89 - logprobs=logprobs,
90 - sample_indices=sample_indices,
91 - seeds=seeds,
92 - max_best_of=max_best_of,
93 - modify_greedy_probs=modify_greedy_probs,
94 - save_logprobs=save_logprobs,
95 - _save_modified_probs=True)
96 - assert sampled_tokens.shape == (bs, max_best_of)
97 - for i in range(bs):
98 - assert torch.all(sampled_tokens[i] == i * (vocab_size // bs))
99 - request_uses_random_sampling = random_sampling_mask[0, i]
100 - if modify_greedy_probs and not request_uses_random_sampling:
101 - # If we are modifying greedy probs and the request is greedy,
102 - # we want to make sure the probs tensor is modified in place
103 - torch.testing.assert_close(
104 - probs[i][sampled_tokens[i]],
105 - torch.full_like(probs[i][sampled_tokens[i]], 1.0))
106 - assert torch.sum(probs[i]) == 1.0
107 - torch.testing.assert_close(
108 - sampled_modified_probs[i][0],
109 - torch.full_like(sampled_modified_probs[i][0], 1.0))
110 - elif request_uses_random_sampling:
111 - # If the request is random, we want to make sure
112 - # sampled_modified_probs tensor has noise added
113 - # (and thus is different from probs tensor)
114 - assert not torch.allclose(sampled_modified_probs[i][0],
115 - probs[i][sampled_tokens[i]])
116 - elif not request_uses_random_sampling:
117 - # If the request is greedy and we are not modifying greedy probs,
118 - # we want to make sure sampled_modified_probs tensor is the same as
119 - # the probs tensor.
120 - torch.testing.assert_close(sampled_modified_probs[i],
121 - probs[i][sampled_tokens[i]])
122 -
123 - if save_logprobs:
124 - assert sampled_logprobs.shape == (bs, max_best_of)
125 - for i in range(bs):
126 - for best_of in range(max_best_of):
127 - assert torch.all(sampled_logprobs[i] == logprobs[i][
128 - sampled_tokens[i, best_of]])
129 - else:
130 - assert sampled_logprobs is None
131 -
132 -
133 -@pytest.mark.parametrize("random_sampling", [True, False, "mixed"])
134 -@pytest.mark.parametrize("max_best_of", [1, 2, 3, 4, 5])
135 -@pytest.mark.parametrize("modify_greedy_probs", [True, False])
136 -@pytest.mark.parametrize("seed", [1337])
137 -@pytest.mark.parametrize("vocab_size",
138 - [SINGLE_SPLIT_VOCAB_SIZE, MULTI_SPLIT_VOCAB_SIZE])
139 -def test_sample_prompt_logprobs(random_sampling, max_best_of,
140 - modify_greedy_probs, seed, vocab_size):
141 -
142 - set_random_seed(seed)
143 - prompt_sizes = [16, 32, 64, 128] * 2
144 - samples = 8
145 - bs = samples + sum(prompt_sizes)
146 - probs = torch.zeros((bs, vocab_size), dtype=torch.float32, device="cuda")
147 - for i in range(bs):
148 - probs[i, i * (vocab_size // bs)] = 1.0
149 - logprobs = torch.rand_like(probs)
150 - sample_indices = torch.tensor(prompt_sizes,
151 - dtype=torch.long,
152 - device="cuda").cumsum_(0)
153 - n_splits = get_num_triton_sampler_splits(probs.shape[1])
154 - if random_sampling == "mixed":
155 - random_sampling_mask = torch.rand(
156 - (n_splits, samples), device="cuda") < 0.5
157 - elif random_sampling:
158 - random_sampling_mask = torch.ones((n_splits, samples),
159 - dtype=torch.bool,
160 - device="cuda")
161 - else:
162 - random_sampling_mask = torch.zeros((n_splits, samples),
163 - dtype=torch.bool,
164 - device="cuda")
165 -
166 - seeds = torch.randint(1,
167 - torch.iinfo(torch.long).max, (n_splits, samples),
168 - device="cuda").mul_(random_sampling_mask)
169 - #ditto
170 - with patch("vllm.model_executor.layers.ops.sample._sample_triton",
171 - LibEntry(_sample_triton)):
172 - sampled_tokens, sampled_logprobs, _ = sample(
173 - probs=probs,
174 - logprobs=logprobs,
175 - sample_indices=sample_indices,
176 - seeds=seeds,
177 - max_best_of=max_best_of,
178 - modify_greedy_probs=modify_greedy_probs,
179 - save_logprobs=True)
180 - assert sampled_tokens.shape == (samples, max_best_of)
181 - assert sampled_logprobs.shape == (samples, max_best_of)
182 - for i, t in enumerate(sample_indices):
183 - assert torch.all(sampled_tokens[i] == t * (vocab_size // bs))
184 - for best_of in range(max_best_of):
185 - assert torch.all(sampled_logprobs[i] == logprobs[sample_indices[i]]
186 - [sampled_tokens[i, best_of]])
187 -
188 -
189 -@pytest.mark.parametrize("seed", list(range(16)))
190 -def test_get_sequence_seeds(seed):
191 - """Ensure that we get a different child seed from base
192 - seed + extra entropy"""
193 - starting_seed = seed
194 - seq_seed = None
195 - extra_entropy = 1
196 - for i in range(512):
197 - new_seq_seed = SamplingTensors._get_sequence_seeds(starting_seed,
198 - i,
199 - seeds_to_generate=1,
200 - is_greedy=False)[0]
201 - new_seq_seed_extra_entropy = SamplingTensors._get_sequence_seeds(
202 - starting_seed,
203 - i,
204 - extra_entropy,
205 - seeds_to_generate=1,
206 - is_greedy=False)[0]
207 - assert new_seq_seed_extra_entropy != new_seq_seed
208 - assert seq_seed != new_seq_seed
209 - seq_seed = new_seq_seed
vllm/model_executor/layers/ops/__init__.py
vllm/model_executor/layers/ops/rand.py deleted
-157
@@ -1,157 +0,0 @@
1 -from typing import Optional, Union
2 -
3 -import torch
4 -import triton
5 -import triton.language as tl
6 -
7 -
8 -def seeded_uniform(
9 - *size,
10 - seeds: torch.Tensor,
11 - out: Optional[torch.Tensor] = None,
12 - dtype: Optional[torch.dtype] = None,
13 - device: Optional[Union[torch.device, str]] = None,
14 - pin_memory: Optional[bool] = False,
15 -) -> torch.Tensor:
16 - """Similar to torch.rand, but allows for seeds to be set per row.
17 -
18 - seeds must be a 1d tensor. The output tensor may be 1d, 2d, or 3d.
19 - If it is 3d, the additional seeds needed will be derived automatically
20 - in a deterministic fashion:
21 - [
22 - row 0: [columns_with_seed_0], [columns_with_seed0^1], ...
23 - ]
24 - """
25 - n_dims = len(size)
26 -
27 - if n_dims > 3:
28 - raise ValueError("seeded_uniform only supports up to 3D tensors")
29 -
30 - if out is None:
31 - out = torch.empty(*size,
32 - dtype=dtype,
33 - device=device,
34 - pin_memory=pin_memory)
35 - elif out.shape != size:
36 - raise ValueError("shape of out and size must be the same")
37 -
38 - if n_dims == 3:
39 - n_rows, n_3d, n_cols = out.shape
40 - stride_row = out.stride(0)
41 - stride_3d = out.stride(1)
42 - elif n_dims == 2:
43 - n_rows, n_cols = out.shape
44 - n_3d = 1
45 - stride_row = out.stride(0)
46 - stride_3d = 1
47 - else:
48 - n_cols = out.shape[0]
49 - n_rows = 1
50 - n_3d = 1
51 - stride_row = 1
52 - stride_3d = 1
53 -
54 - if seeds.ndim != 1:
55 - raise ValueError("seeds must be a 1D tensor")
56 -
57 - if seeds.numel() != n_rows:
58 - raise ValueError(
59 - "seeds must have the same number of elements as out has rows")
60 -
61 - # The philox PRNG Triton uses generates 4 random numbers at once.
62 - # Therefore, the most efficient use of it is to divide the
63 - # block size by 4, and then save the generated random numbers to
64 - # each of the 4 slices of the tensor.
65 - full_block_size = triton.next_power_of_2(n_cols)
66 - philox_block_size = max(full_block_size // 4, 1)
67 - n_slices = full_block_size // philox_block_size
68 - num_warps = 4
69 - # Manual tuning. This seems to give best performance on A100 for
70 - # simple kernels like this.
71 - if philox_block_size >= 8192:
72 - num_warps = 32
73 - elif philox_block_size >= 4096:
74 - num_warps = 16
75 - elif philox_block_size >= 2048:
76 - num_warps = 8
77 -
78 - _seeded_uniform_triton[(n_rows, n_3d)](
79 - out,
80 - seeds,
81 - stride_row,
82 - stride_3d,
83 - seeds.stride(0),
84 - n_rows,
85 - n_3d,
86 - n_cols,
87 - n_slices=n_slices,
88 - num_warps=num_warps,
89 - block_size=philox_block_size,
90 - )
91 - return out
92 -
93 -
94 -@triton.jit
95 -def _seeded_uniform_triton(
96 - out_ptr: torch.Tensor,
97 - seed_ptr: torch.Tensor,
98 - out_row_stride: int,
99 - out_3d_stride: int,
100 - seed_row_stride: int,
101 - n_rows: int,
102 - n_3d: int,
103 - n_cols: int,
104 - n_slices: tl.constexpr,
105 - block_size: tl.constexpr,
106 -):
107 - """
108 - Generate a random float32 number in [0, 1) for each element in the output
109 - tensor. The random numbers in a row generated using the seed for that row.
110 -
111 - Args:
112 - out_ptr: The output tensor.
113 - seed_ptr: The per-row seeds to use for random number generation.
114 - out_row_stride: The stride between rows of the output tensor.
115 - out_3d_stride: The stride between 3D slices of the output tensor.
116 - seed_row_stride: The stride between rows of the seed tensor.
117 - n_rows: The number of rows in the output tensor.
118 - n_3d: The size of second dimension of the output tensor,
119 - if output tensor is 3D.
120 - n_cols: The number of columns in the output tensor.
121 - n_slices: The number of philox outputs to use.
122 - """
123 - tl.static_assert(n_slices > 0 and n_slices <= 4, "0 < n_slices <= 4")
124 -
125 - # Get the row index.
126 - row_idx = tl.program_id(axis=0)
127 - three_d_idx = tl.program_id(axis=1)
128 -
129 - philox_offsets = tl.arange(0, block_size)
130 - # Get the seed for the current element.
131 - seed = tl.load(seed_ptr + row_idx * seed_row_stride)
132 - if three_d_idx > 0:
133 - seed ^= three_d_idx
134 - # Generate random numbers in [0, 1).
135 - out1, out2, out3, out4 = tl.rand4x(seed, philox_offsets)
136 -
137 - output_row_start_ptr = (out_ptr + row_idx * out_row_stride +
138 - three_d_idx * out_3d_stride)
139 - out1_offsets = philox_offsets
140 - tl.store(output_row_start_ptr + out1_offsets,
141 - out1,
142 - mask=out1_offsets < n_cols)
143 - if n_slices > 1:
144 - out2_offsets = tl.arange(block_size, block_size * 2)
145 - tl.store(output_row_start_ptr + out2_offsets,
146 - out2,
147 - mask=out2_offsets < n_cols)
148 - if n_slices > 2:
149 - out3_offsets = tl.arange(block_size * 2, block_size * 3)
150 - tl.store(output_row_start_ptr + out3_offsets,
151 - out3,
152 - mask=out3_offsets < n_cols)
153 - if n_slices > 3:
154 - out4_offsets = tl.arange(block_size * 3, block_size * 4)
155 - tl.store(output_row_start_ptr + out4_offsets,
156 - out4,
157 - mask=out4_offsets < n_cols)
vllm/model_executor/layers/ops/sample.py deleted
-394
@@ -1,394 +0,0 @@
1 -from typing import Optional, Tuple
2 -
3 -import torch
4 -import triton
5 -import triton.language as tl
6 -
7 -from vllm.model_executor.layers.ops.rand import seeded_uniform
8 -from vllm.triton_utils.sample import get_num_triton_sampler_splits
9 -
10 -_EPS: tl.constexpr = 1e-6
11 -
12 -
13 -def _multi_split_sample(
14 - probs: torch.Tensor,
15 - seeds: torch.Tensor,
16 - n_splits: int,
17 - sampled_tokens_size: Tuple[int, int],
18 - sampled_logprobs_size: Tuple[int, int],
19 - sample_indices: torch.Tensor,
20 - logprobs: torch.Tensor,
21 - *,
22 - modify_greedy_probs: bool = False,
23 - save_logprobs: bool = False,
24 -):
25 - """Sample tokens where vocab size is split into multiple parts
26 - (too large for Triton otherwise)."""
27 - assert seeds.ndim == 2 and seeds.shape[0] == n_splits
28 - split_probs = probs.tensor_split(n_splits, 1)
29 - split_logprobs = logprobs.tensor_split(n_splits, 1)
30 - sampled_tokens_tmp = [
31 - torch.empty(sampled_tokens_size, dtype=torch.long, device=probs.device)
32 - for _ in range(n_splits)
33 - ]
34 - sampled_logprobs_tmp = [
35 - torch.empty(sampled_logprobs_size,
36 - dtype=probs.dtype,
37 - device=probs.device) for _ in range(n_splits)
38 - ]
39 - # We are purposefuly using sampled_tokens_size as we need to always
40 - # save modified probs in this case.
41 - sampled_modified_probs_tmp = [
42 - torch.empty(sampled_tokens_size,
43 - dtype=probs.dtype,
44 - device=probs.device) for _ in range(n_splits)
45 - ]
46 - for i in range(n_splits):
47 - n_samples = sample_indices.shape[0]
48 - n_cols = split_probs[i].shape[1]
49 - n_best = sampled_tokens_tmp[i].shape[1]
50 - uniform_noise = seeded_uniform(n_samples,
51 - n_best,
52 - n_cols,
53 - seeds=seeds[i].flatten(),
54 - device=split_probs[i].device,
55 - dtype=split_probs[i].dtype)
56 - # TODO(yard1): See if we can remove the contiguous() calls.
57 - # Will need kernel support.
58 - _sample(
59 - split_probs[i].contiguous(),
60 - split_logprobs[i].contiguous(),
61 - sample_indices,
62 - sampled_tokens_tmp[i],
63 - sampled_logprobs_tmp[i],
64 - sampled_modified_probs_tmp[i],
65 - seeds[i],
66 - uniform_noise,
67 - modify_greedy_probs=False,
68 - save_logprobs=save_logprobs,
69 - save_modified_probs=True,
70 - )
71 - if i > 0:
72 - # Add offset to sampled tokens
73 - sampled_tokens_tmp[i].add_(i * split_probs[i - 1].shape[1])
74 - sampled_tokens = torch.stack(sampled_tokens_tmp)
75 - sampled_modified_probs = torch.stack(sampled_modified_probs_tmp)
76 - # Reduce the results from the splits.
77 - sampled_modified_probs, indices = torch.max(sampled_modified_probs,
78 - dim=0,
79 - keepdim=True)
80 - sampled_tokens = sampled_tokens.gather(0, indices).squeeze(0)
81 - if save_logprobs:
82 - sampled_logprobs = torch.stack(sampled_logprobs_tmp)
83 - sampled_logprobs = sampled_logprobs.gather(0, indices).squeeze(0)
84 - else:
85 - sampled_logprobs = None
86 - sampled_modified_probs = sampled_modified_probs.squeeze(0)
87 -
88 - if modify_greedy_probs:
89 - # We need to modify the greedy probs for the sampled tokens.
90 - # We can't do this in the kernel as we need to know the
91 - # sampled tokens.
92 - probs.fill_(0.0)
93 - probs.scatter_(1, sampled_tokens, 1.0)
94 -
95 - return (sampled_tokens, sampled_logprobs, sampled_modified_probs)
96 -
97 -
98 -def sample(
99 - probs: torch.Tensor,
100 - seeds: torch.Tensor,
101 - *,
102 - max_best_of: int = 1,
103 - sample_indices: Optional[torch.Tensor] = None,
104 - logprobs: Optional[torch.Tensor] = None,
105 - modify_greedy_probs: bool = False,
106 - save_logprobs: bool = False,
107 - _save_modified_probs: bool = False, # pylint: disable=invalid-name
108 -) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]:
109 - """Sample tokens from probs. with per-sequence seeds.
110 -
111 - Can sample from a subset of sequences through sample_indices.
112 -
113 - Args:
114 - probs: Probabilities to sample from.
115 - shape = [batch_size, vocab_size]
116 - seeds: Per-sequence seed values.
117 - shape = [n, math.ceil(vocab_size / MAX_TRITON_N_COLS)]
118 - max_best_of: Number of samples to generate per sequence.
119 - Sequence seed will be incremented by 1 each time.
120 - sample_indices: Indices of sequences to sample from.
121 - If not provided, will sample from all sequences.
122 - shape = [n]
123 - logprobs: Log-probabilities of the sampled tokens.
124 - Only used for saving the logprobs if save_logprobs is True.
125 - shape = [batch_size, vocab_size]
126 - modify_greedy_probs: Whether to modify the greedy probabilities
127 - for speculative sampling (sampled token = 1.0,
128 - everything else = 0.0).
129 - save_logprobs: Whether to save the log-probabilities of the
130 - sampled tokens to a tensor.
131 - _save_modified_probs: Whether to save the modified probabilities
132 - (including gumbel noise) of the sampled tokens to a tensor.
133 - DOES NOT include the modification done by modify_greedy_probs
134 - (because we want to use the unmodified probs to pick the best
135 - split in case of multi-split sampling).
136 - This is exposed only for testing.
137 -
138 - Returns:
139 - sampled_tokens: shape = [n, max_best_of]
140 - sampled_logprobs: shape = [n, max_best_of] if save_logprobs else None
141 - sampled_modified_probs: shape = [n, max_best_of]
142 - if save_modified_probs else None
143 - """
144 - if sample_indices is None:
145 - sample_indices = torch.arange(0, probs.shape[0], device=probs.device)
146 -
147 - sampled_tokens_size = (sample_indices.size(0), max_best_of)
148 - if save_logprobs:
149 - if logprobs is None:
150 - raise ValueError(
151 - "logprobs tensor must be provided if save_logprobs is True")
152 - sampled_logprobs_size = sampled_tokens_size
153 - else:
154 - # Empty tensors to invoke the kernel
155 - sampled_logprobs_size = (0, 0)
156 - logprobs = probs
157 -
158 - assert logprobs is not None
159 - if _save_modified_probs:
160 - sampled_modified_probs_size = sampled_tokens_size
161 - else:
162 - # Empty tensors to invoke the kernel
163 - sampled_modified_probs_size = (0, 0)
164 -
165 - # If the number of columns in probs is too large for Triton to handle,
166 - # we split the tensor and sample from each split separately, and then
167 - # do an argmax+gather to combine the results.
168 - n_splits = get_num_triton_sampler_splits(probs.shape[1])
169 - if n_splits > 1:
170 - (sampled_tokens, sampled_logprobs,
171 - sampled_modified_probs) = _multi_split_sample(
172 - probs,
173 - seeds,
174 - n_splits,
175 - sampled_tokens_size,
176 - sampled_logprobs_size,
177 - sample_indices,
178 - logprobs=logprobs,
179 - modify_greedy_probs=modify_greedy_probs,
180 - save_logprobs=save_logprobs)
181 - else:
182 - sampled_tokens = torch.empty(sampled_tokens_size,
183 - dtype=torch.long,
184 - device=probs.device)
185 - sampled_logprobs = torch.empty(sampled_logprobs_size,
186 - dtype=probs.dtype,
187 - device=probs.device)
188 - sampled_modified_probs = torch.empty(sampled_modified_probs_size,
189 - dtype=probs.dtype,
190 - device=probs.device)
191 - n_samples = sample_indices.shape[0]
192 - n_cols = probs.shape[1]
193 - uniform_noise = seeded_uniform(n_samples,
194 - max_best_of,
195 - n_cols,
196 - seeds=seeds.flatten(),
197 - device=probs.device,
198 - dtype=probs.dtype)
199 -
200 - _sample(
201 - probs,
202 - logprobs,
203 - sample_indices,
204 - sampled_tokens,
205 - sampled_logprobs,
206 - sampled_modified_probs,
207 - seeds,
208 - uniform_noise,
209 - modify_greedy_probs=modify_greedy_probs,
210 - save_logprobs=save_logprobs,
211 - save_modified_probs=_save_modified_probs,
212 - )
213 - return (sampled_tokens, sampled_logprobs if save_logprobs else None,
214 - sampled_modified_probs if _save_modified_probs else None)
215 -
216 -
217 -def _sample(probs: torch.Tensor,
218 - logprobs: torch.Tensor,
219 - sample_indices: torch.Tensor,
220 - output_samples: torch.Tensor,
221 - output_logprobs: torch.Tensor,
222 - output_modified_probs: torch.Tensor,
223 - seeds: torch.Tensor,
224 - uniform_noise: torch.Tensor,
225 - *,
226 - modify_greedy_probs: bool = False,
227 - save_logprobs: bool = True,
228 - save_modified_probs: bool = False) -> torch.Tensor:
229 - """Sample tokens from probs.
230 -
231 - Args:
232 - probs [batch_size, vocab_size]: probs to sample from.
233 - logprobs [batch_size, vocab_size]: logprobs (used when
234 - save_logprobsis True).
235 - sample_indices [n]: Indices of the samples to use for each row of probs.
236 - output_samples [n, n_best]: Output tensor to store samples in.
237 - output_logprobs [n, n_best]: Output tensor to store logprobs in.
238 - output_modified_probs [n, n_best]: Output tensor to store
239 - probs of chosen tokens in (modified with noise).
240 - seeds [n]: Seeds to use for sampling. If the seed is 0, we use
241 - greedy sampling. Note this is ONLY used for determining
242 - whether to use random sampling or not. The actual random
243 - noise should be passed as uniform_noise.
244 - uniform_noise [batch_size, n_best, vocab_size]: Uniform
245 - noise to use for random sampling (will be converted
246 - to exponential gumbel noise by the kernel).
247 - modify_greedy_probs: If True, we modify the probs tensor in-place
248 - to encode the sampling method used for each row. This is used
249 - in speculative decoding. Only applies in greedy decoding.
250 - save_logprobs: If True, we save the logprobs of the sampled tokens
251 - in the output_logprobs tensor.
252 - save_modified_probs: If True, we save the modified probs (with noise)
253 - of the sampled tokens in the output_modified_probs tensor.
254 - DOES NOT include the modification done by modify_greedy_probs
255 - (because we want to use the unmodified probs to pick the best
256 - split in case of multi-split sampling).
257 - """
258 - n_samples = sample_indices.shape[0]
259 - n_cols = probs.shape[1]
260 - n_best = output_samples.shape[1] if len(output_samples.shape) > 1 else 1
261 -
262 - # The block size is the smallest power of two greater than the number of
263 - # columns in probs
264 - block_size = triton.next_power_of_2(n_cols)
265 - num_warps = 4
266 - # Manual tuning. This seems to give best performance on A100 for
267 - # simple kernels like this.
268 - if block_size >= 8192:
269 - num_warps = 32
270 - elif block_size >= 4096:
271 - num_warps = 16
272 - elif block_size >= 2048:
273 - num_warps = 8
274 -
275 - # Enqueue kernel. The 1D launch grid is simple: we have one kernel
276 - # instance per row of the probs matrix
277 - _sample_triton[(n_samples, n_best)](
278 - sample_indices,
279 - output_samples,
280 - output_logprobs,
281 - output_modified_probs,
282 - probs,
283 - logprobs,
284 - seeds,
285 - uniform_noise,
286 - output_samples.stride(0),
287 - probs.stride(0),
288 - uniform_noise.stride(0),
289 - uniform_noise.stride(1) if n_best > 1 else 1,
290 - n_samples,
291 - n_cols,
292 - n_best,
293 - num_warps=num_warps,
294 - block_size=block_size,
295 - modify_greedy_probs=modify_greedy_probs,
296 - save_logprobs=save_logprobs,
297 - save_modified_probs=save_modified_probs,
298 - )
299 - return output_samples, output_logprobs, output_modified_probs
300 -
301 -
302 -@triton.jit
303 -def _uniform_to_exponential(uniform_noise):
304 - """Convert uniform samples to exponential samples."""
305 - # tl.rand returns values in [0, 1), so we clamp lower bound
306 - # to _EPS to avoid log(0) and thus division by 0 later
307 - lb = tl.full(uniform_noise.shape, _EPS, uniform_noise.dtype)
308 - uniform_noise = tl.maximum(uniform_noise, lb)
309 - # Use the inversion method to turn uniform samples
310 - # into exponential samples
311 - exponential_noise = -tl.log(uniform_noise)
312 - return exponential_noise
313 -
314 -
315 -@triton.jit
316 -def _sample_triton(
317 - sample_indices_ptr: torch.Tensor, output_ptr: torch.Tensor,
318 - output_logprobs_ptr: torch.Tensor,
319 - output_modified_probs_ptr: torch.Tensor, probs_ptr: torch.Tensor,
320 - logprobs_ptr: torch.Tensor, seeds_ptr: torch.Tensor,
321 - uniform_noise_ptr: torch.Tensor, output_row_stride: int,
322 - probs_row_stride: int, uniform_noise_row_stride: int,
323 - uniform_noise_best_stride: int, n_samples: int, n_cols: int,
324 - n_best: int, block_size: tl.constexpr,
325 - modify_greedy_probs: tl.constexpr, save_logprobs: tl.constexpr,
326 - save_modified_probs: tl.constexpr):
327 - # The rows are independent, so we parallelize across those
328 - sample_idx = tl.program_id(0)
329 - best_idx = tl.program_id(1)
330 -
331 - # Load the row index from DRAM
332 - row_idx = tl.load(sample_indices_ptr + sample_idx)
333 - seed = tl.load(seeds_ptr + sample_idx)
334 - uses_random_sampling = seed != 0
335 -
336 - # The stride represents how much we need to increase the
337 - # pointer to advance 1 row
338 - row_start_ptr = probs_ptr + row_idx * probs_row_stride
339 -
340 - # The block size is the next power of two greater than n_cols,
341 - # so we can fit each row in a single block
342 - col_offsets = tl.arange(0, block_size)
343 -
344 - # Load the row into SRAM, using a mask since block_size may be > than n_cols
345 - row = tl.load(row_start_ptr + col_offsets,
346 - mask=col_offsets < n_cols,
347 - other=float("-inf"))
348 -
349 - if uses_random_sampling:
350 - uniform_noise_start_ptr = (uniform_noise_ptr +
351 - sample_idx * uniform_noise_row_stride +
352 - best_idx * uniform_noise_best_stride)
353 - uniform_noise = tl.load(uniform_noise_start_ptr + col_offsets,
354 - mask=col_offsets < n_cols,
355 - other=0.5)
356 - exponential_noise = _uniform_to_exponential(uniform_noise)
357 - row /= exponential_noise
358 -
359 - sampled_value, sampled_token = tl.max(row, axis=0, return_indices=True)
360 - # clamp sampled token to n_cols - 1
361 - # this should not be necessary, but we do it
362 - # just in case
363 - if sampled_token >= n_cols:
364 - sampled_token = n_cols - 1
365 - # Write back output to DRAM
366 - output_row_start_ptr = (output_ptr + sample_idx * output_row_stride +
367 - best_idx)
368 - tl.store(output_row_start_ptr, sampled_token)
369 -
370 - if modify_greedy_probs: # noqa
371 - if not uses_random_sampling:
372 - # Set the probability of the sampled token to 1, all other
373 - # tokens to zero. This is used in speculative decoding where
374 - # the sampling method must be encoded within the sampled
375 - # probability distributions.
376 - row = tl.where(col_offsets == sampled_token, 1.0, 0.0)
377 - tl.store(row_start_ptr + col_offsets,
378 - row,
379 - mask=col_offsets < n_cols)
380 -
381 - if save_modified_probs:
382 - output_row_start_ptr = (output_modified_probs_ptr +
383 - sample_idx * output_row_stride + best_idx)
384 - tl.store(output_row_start_ptr, sampled_value)
385 -
386 - if save_logprobs:
387 - # Load the row into SRAM, using a mask since block_size
388 - # may be > than n_cols
389 - sampled_logprob = tl.load(logprobs_ptr + row_idx * probs_row_stride +
390 - sampled_token)
391 - # Write back output to DRAM
392 - output_row_start_ptr = (output_logprobs_ptr +
393 - sample_idx * output_row_stride + best_idx)
394 - tl.store(output_row_start_ptr, sampled_logprob)
vllm/model_executor/layers/sampler.py
+3 -94
@@ -10,12 +10,6 @@ import msgspec
10 import torch
11 import torch.nn as nn
12
13 -from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
14 -from vllm.triton_utils import HAS_TRITON
15 -
16 -if HAS_TRITON:
17 - from vllm.model_executor.layers.ops.sample import sample as sample_triton
18 -
13 import vllm.envs as envs
14 from vllm.model_executor.sampling_metadata import (SamplingMetadata,
15 SamplingTensors,
@@ -23,6 +17,7 @@ from vllm.model_executor.sampling_metadata import (SamplingMetadata,
17 from vllm.sampling_params import SamplingType
18 from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
19 PromptLogprobs, SampleLogprobs, SequenceOutput)
20 +from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
21
22 if envs.VLLM_USE_FLASHINFER_SAMPLER and find_spec("flashinfer"):
23 import flashinfer.sampling
@@ -740,7 +735,7 @@ def _sample_with_torch(
735 ) -> SampleReturnType:
736 '''Torch-oriented _sample() implementation.
737
743 - Single-step scheduling:
738 + Single-step scheduling:
739 * Perform GPU-side sampling computation
740 * Immediately Pythonize sampling result
741
@@ -777,7 +772,7 @@ def _sample_with_torch(
772 # Counterintiutively, having two loops here is actually faster.
773 # The first loop can run without waiting on GPU<->CPU sync.
774 for sampling_type in SamplingType:
780 - sample_indices = categorized_sample_indices[sampling_type][:, 0]
775 + sample_indices = categorized_sample_indices[sampling_type]
776 num_tokens = len(sample_indices)
777 if num_tokens == 0:
778 continue
@@ -863,88 +858,6 @@ def _sample_with_torch(
858 )
859
860
866 -def _sample_with_triton_kernel(
867 - probs: torch.Tensor,
868 - logprobs: torch.Tensor,
869 - sampling_metadata: SamplingMetadata,
870 - sampling_tensors: SamplingTensors,
871 -) -> SampleResultType:
872 - categorized_seq_group_ids: Dict[SamplingType,
873 - List[int]] = {t: []
874 - for t in SamplingType}
875 - categorized_sample_indices = sampling_metadata.categorized_sample_indices
876 - for i, seq_group in enumerate(sampling_metadata.seq_groups):
877 - sampling_params = seq_group.sampling_params
878 - sampling_type = sampling_params.sampling_type
879 - categorized_seq_group_ids[sampling_type].append(i)
880 -
881 - sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
882 - sample_metadata: Dict[SamplingType,
883 - Tuple[List[int], List[SequenceGroupToSample],
884 - torch.Tensor, torch.Tensor]] = {}
885 - max_best_of_in_batch = 1
886 -
887 - # Counterintiutively, having two loops here is actually faster.
888 - # The first loop can run without waiting on GPU<->CPU sync.
889 - for sampling_type in SamplingType:
890 - sample_indices = categorized_sample_indices[sampling_type][:, 0]
891 - sampled_token_indices = categorized_sample_indices[sampling_type][:, 1]
892 - num_tokens = len(sample_indices)
893 - if num_tokens == 0:
894 - continue
895 - seq_group_id = categorized_seq_group_ids[sampling_type]
896 - seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_id]
897 - sample_metadata[sampling_type] = (seq_group_id, seq_groups,
898 - sample_indices,
899 - sampled_token_indices)
900 - if sampling_type in (SamplingType.GREEDY, SamplingType.RANDOM,
901 - SamplingType.RANDOM_SEED):
902 - for seq_group in seq_groups:
903 - if seq_group.is_prompt:
904 - sampling_params = seq_group.sampling_params
905 - max_best_of_in_batch = max(max_best_of_in_batch,
906 - sampling_params.best_of)
907 - elif sampling_type == SamplingType.BEAM:
908 - beam_search_logprobs = logprobs[sample_indices]
909 - else:
910 - raise ValueError(f"Unsupported sampling type: {sampling_type}")
911 -
912 - sampled_tokens, _, _ = sample_triton(
913 - probs=probs,
914 - seeds=sampling_tensors.sampling_seeds,
915 - max_best_of=max_best_of_in_batch,
916 - sample_indices=sampling_tensors.sample_indices,
917 - logprobs=logprobs,
918 - # don't save logprobs because we have logic for that below
919 - # TODO: use this instead of the CPU-based logic below
920 - save_logprobs=False,
921 - )
922 -
923 - # GPU<->CPU sync happens in the loop below.
924 -
925 - for sampling_type in SamplingType:
926 - if sampling_type not in sample_metadata:
927 - continue
928 - (seq_group_id, seq_groups, sample_indices,
929 - sampled_token_indices) = sample_metadata[sampling_type]
930 - if sampling_type == SamplingType.GREEDY:
931 - sample_results = _greedy_sample(
932 - seq_groups, sampled_tokens[sampled_token_indices][:, 0])
933 - elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
934 - sample_results = _random_sample(
935 - seq_groups, sampled_tokens[sampled_token_indices])
936 - elif sampling_type == SamplingType.BEAM:
937 - sample_results = _beam_search_sample(seq_groups,
938 - beam_search_logprobs)
939 - sample_results_dict.update(zip(seq_group_id, sample_results))
940 -
941 - sample_results = [
942 - sample_results_dict.get(i, ([], []))
943 - for i in range(len(sampling_metadata.seq_groups))
944 - ]
945 - return sample_results
946 -
947 -
861 def _sample(
862 probs: torch.Tensor,
863 logprobs: torch.Tensor,
@@ -974,10 +887,6 @@ def _sample(
887 modify_greedy_probs=modify_greedy_probs,
888 )
889
977 - # TODO: Enable once Triton kernel & associated code is faster.
978 - # return _sample_with_triton_kernel(probs, logprobs, sampling_metadata,
979 - # sampling_tensors)
980 -
890
891 def _get_ranks(x: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
892 """
vllm/model_executor/sampling_metadata.py
+58 -153
@@ -1,4 +1,3 @@
1 -import random
1 from array import array
2 from dataclasses import dataclass
3 from typing import Dict, List, Optional, Tuple
@@ -8,15 +7,10 @@ import torch
7 from vllm.sampling_params import SamplingParams, SamplingType
8 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData,
9 SequenceGroupMetadata)
11 -from vllm.triton_utils.sample import get_num_triton_sampler_splits
10 from vllm.utils import (PyObjectCache, async_tensor_h2d,
13 - is_pin_memory_available, make_tensor_with_pad,
14 - maybe_expand_dim)
11 + is_pin_memory_available, make_tensor_with_pad)
12
13 _SAMPLING_EPS = 1e-5
17 -_SEED_0_REPLACEMENT = 3403598558
18 -# Some triton sampler related code is guarded before it is ready.
19 -_USE_TRITON_SAMPLER = False
14
15
16 @dataclass
@@ -74,12 +68,12 @@ def gen_seq_group_to_sample_builder(num_seqs: int):
68 generator=None,
69 is_prompt=True,
70 prompt_logprob_indices=[],
77 - sample_indices=[])
71 + sample_indices=[],
72 + )
73
74
75 class SamplingMetadataCache:
81 - """Used to cache SamplingMetadata objects between scheduler iterations
82 - """
76 + """Used to cache SamplingMetadata objects between scheduler iterations"""
77
78 def __init__(self):
79 self._seq_group_to_sample_cache: Dict[int, PyObjectCache] = {}
@@ -124,12 +118,12 @@ class SamplingMetadata:
118 The first tuple is [1, 2] (sampled index within original logit),
119 and the second tuple is [0, 1] (sampled index within pruned logit).
120 num_prompts: Number of prompt sequence groups in seq_groups.
127 - skip_sampler_cpu_output: Indicates if we want to skip the GPU=>CPU
121 + skip_sampler_cpu_output: Indicates if we want to skip the GPU=>CPU
122 serialization of token outputs.
129 - reuse_sampling_tensors: Indicates if we want to reuse sampling
123 + reuse_sampling_tensors: Indicates if we want to reuse sampling
124 tensors that are part of the sampler forward pass. Currently,
125 it is mainly used for multi-step decode.
132 -
126 +
127 """
128
129 def __init__(
@@ -165,16 +159,19 @@ class SamplingMetadata:
159 num_prompts,
160 ) = _prepare_seq_groups(seq_group_metadata_list, seq_lens, query_lens,
161 device, generators, cache)
168 - selected_token_indices = async_tensor_h2d(selected_token_indices,
169 - dtype=torch.long,
170 - target_device=device,
171 - pin_memory=pin_memory)
162 + selected_token_indices = async_tensor_h2d(
163 + selected_token_indices,
164 + dtype=torch.long,
165 + target_device=device,
166 + pin_memory=pin_memory,
167 + )
168 categorized_sample_indices = {
173 - t: maybe_expand_dim(
174 - async_tensor_h2d(seq_ids,
175 - dtype=torch.int,
176 - target_device=device,
177 - pin_memory=pin_memory), 2, 2)
169 + t: async_tensor_h2d(
170 + seq_ids,
171 + dtype=torch.int,
172 + target_device=device,
173 + pin_memory=pin_memory,
174 + )
175 for t, seq_ids in categorized_sample_indices.items()
176 }
177
@@ -201,8 +198,8 @@ def _prepare_seq_groups(
198 device: str,
199 generators: Optional[Dict[str, torch.Generator]] = None,
200 cache: Optional[SamplingMetadataCache] = None,
204 -) -> Tuple[List[SequenceGroupToSample], List[int], Dict[
205 - SamplingType, List[Tuple[int, int]]], int]:
201 +) -> Tuple[List[SequenceGroupToSample], List[int], Dict[SamplingType,
202 + List[int]], int, ]:
203 """Prepare sequence groups and indices for sampling.
204
205 Args:
@@ -233,16 +230,13 @@ def _prepare_seq_groups(
230 # Sampling type -> (
231 # indices to sample/prompt logprob within pruned output logits,
232 # indices to sample within pruned logits)
236 - categorized_sample_indices: Dict[SamplingType, List[Tuple[int, int]]] = {
233 + categorized_sample_indices: Dict[SamplingType, List[int]] = {
234 t: []
235 for t in SamplingType
236 }
237 # Index of logits to compute logprob. Logits include both prompt logprob
238 # and sample logprob indices.
239 logit_idx = 0
243 - # Index to sample from a sample tensor. It is used by triton sample kernel.
244 - # See `_sample_with_triton_kernel` for more details.
245 - sample_idx = 0
240 # Total number of prompts from given sequence groups.
241 num_prompts = 0
242
@@ -264,10 +258,10 @@ def _prepare_seq_groups(
258 # If the current seq group is in decode stage, it is None.
259 seq_len: Optional[int] = None
260 query_len: Optional[int] = None
267 - prompt_logprob_indices: List[int] = \
268 - sample_obj.prompt_logprob_indices if cache is not None else []
269 - sample_indices: List[int] = \
270 - sample_obj.sample_indices if cache is not None else []
261 + prompt_logprob_indices: List[int] = (sample_obj.prompt_logprob_indices
262 + if cache is not None else [])
263 + sample_indices: List[int] = (sample_obj.sample_indices
264 + if cache is not None else [])
265 do_sample = seq_group_metadata.do_sample
266
267 if seq_group_metadata.is_prompt:
@@ -333,11 +327,8 @@ def _prepare_seq_groups(
327 if do_sample:
328 sample_indices.extend(range(logit_idx, logit_idx + sample_len))
329 categorized_sample_indices[sampling_params.sampling_type].extend(
336 - list(
337 - zip(range(logit_idx, logit_idx + sample_len),
338 - range(sample_idx, sample_idx + sample_len))))
330 + list(range(logit_idx, logit_idx + sample_len)))
331 logit_idx += sample_len
340 - sample_idx += sample_len
332
333 if cache is not None:
334 sample_obj.sampling_params = sampling_params
@@ -356,7 +347,8 @@ def _prepare_seq_groups(
347 generator=generator,
348 is_prompt=is_prompt,
349 prompt_logprob_indices=list(prompt_logprob_indices),
359 - sample_indices=list(sample_indices))
350 + sample_indices=list(sample_indices),
351 + )
352
353 seq_groups.append(sample_obj)
354
@@ -378,9 +370,6 @@ class SamplingTensors:
370 presence_penalties: torch.Tensor
371 frequency_penalties: torch.Tensor
372 repetition_penalties: torch.Tensor
381 - sampling_seeds: torch.Tensor
382 - sample_indices: torch.Tensor
383 - extra_seeds: Optional[torch.Tensor]
373 prompt_tokens: torch.Tensor
374 output_tokens: torch.Tensor
375
@@ -391,15 +380,7 @@ class SamplingTensors:
380 vocab_size: int,
381 device: torch.device,
382 dtype: torch.dtype,
394 - *,
395 - extra_seeds_to_generate: int = 0,
396 - extra_entropy: Optional[Tuple[int, ...]] = None
383 ) -> Tuple["SamplingTensors", bool, bool, bool]:
398 - """
399 - extra_seeds_to_generate: extra seeds to generate using the
400 - user-defined seed for each sequence.
401 - extra_entropy: extra entropy to use when generating seeds.
402 - """
384 prompt_tokens: List[array] = []
385 output_tokens: List[array] = []
386 top_ks: List[int] = []
@@ -409,19 +390,10 @@ class SamplingTensors:
390 presence_penalties: List[float] = []
391 frequency_penalties: List[float] = []
392 repetition_penalties: List[float] = []
412 - sampling_seeds: List[int] = []
413 - sample_indices: List[int] = []
393 do_penalties = False
394 do_top_p_top_k = False
395 do_min_p = False
396
418 - if _USE_TRITON_SAMPLER:
419 - prompt_best_of: List[int] = []
420 -
421 - # We need one base seed per Triton slice.
422 - seeds_to_generate = (extra_seeds_to_generate +
423 - get_num_triton_sampler_splits(vocab_size))
424 -
397 assert sampling_metadata.seq_groups is not None
398 for seq_group in sampling_metadata.seq_groups:
399 seq_ids = seq_group.seq_ids
@@ -452,7 +424,7 @@ class SamplingTensors:
424 do_penalties = True
425
426 is_prompt = seq_group.is_prompt
455 - if (is_prompt and sampling_params.prompt_logprobs is not None):
427 + if is_prompt and sampling_params.prompt_logprobs is not None:
428 # For tokens in the prompt that we only need to get
429 # their logprobs
430 query_len = seq_group.query_len
@@ -477,28 +449,6 @@ class SamplingTensors:
449 frequency_penalties += [f] * len(seq_ids)
450 repetition_penalties += [r] * len(seq_ids)
451
480 - if _USE_TRITON_SAMPLER:
481 - if is_prompt:
482 - prompt_best_of.append(sampling_params.best_of)
483 - query_len = seq_group.query_len
484 - assert query_len is not None
485 -
486 - seed = sampling_params.seed
487 - is_greedy = sampling_params.sampling_type == SamplingType.GREEDY
488 -
489 - for seq_id in seq_ids:
490 - seq_data = seq_group.seq_data[seq_id]
491 - extra_entropy = extra_entropy or ()
492 - seq_seeds = cls._get_sequence_seeds(
493 - seed,
494 - seq_data.get_len(),
495 - *extra_entropy,
496 - seq_id,
497 - seeds_to_generate=seeds_to_generate,
498 - is_greedy=is_greedy)
499 - sampling_seeds.append(seq_seeds)
500 - sample_indices.extend(seq_group.sample_indices)
501 -
452 if do_penalties:
453 for seq_group in sampling_metadata.seq_groups:
454 seq_ids = seq_group.seq_ids
@@ -518,23 +468,37 @@ class SamplingTensors:
468 output_tokens.append(seq_data.output_token_ids_array)
469
470 sampling_tensors = SamplingTensors.from_lists(
521 - temperatures, top_ps, top_ks, min_ps, presence_penalties,
522 - frequency_penalties, repetition_penalties, sampling_seeds,
523 - sample_indices, prompt_tokens, output_tokens, vocab_size,
524 - extra_seeds_to_generate, device, dtype)
471 + temperatures,
472 + top_ps,
473 + top_ks,
474 + min_ps,
475 + presence_penalties,
476 + frequency_penalties,
477 + repetition_penalties,
478 + prompt_tokens,
479 + output_tokens,
480 + vocab_size,
481 + device,
482 + dtype,
483 + )
484 return (sampling_tensors, do_penalties, do_top_p_top_k, do_min_p)
485
486 @classmethod
528 - def from_lists(cls, temperatures: List[float], top_ps: List[float],
529 - top_ks: List[int], min_ps: List[float],
530 - presence_penalties: List[float],
531 - frequency_penalties: List[float],
532 - repetition_penalties: List[float],
533 - sampling_seeds: List[int], sample_indices: List[int],
534 - prompt_tokens: List[array], output_tokens: List[array],
535 - vocab_size: int, extra_seeds_to_generate: int,
536 - device: torch.device,
537 - dtype: torch.dtype) -> "SamplingTensors":
487 + def from_lists(
488 + cls,
489 + temperatures: List[float],
490 + top_ps: List[float],
491 + top_ks: List[int],
492 + min_ps: List[float],
493 + presence_penalties: List[float],
494 + frequency_penalties: List[float],
495 + repetition_penalties: List[float],
496 + prompt_tokens: List[array],
497 + output_tokens: List[array],
498 + vocab_size: int,
499 + device: torch.device,
500 + dtype: torch.dtype,
501 + ) -> "SamplingTensors":
502 # Note that the performance will be very bad without
503 # pinned memory.
504 pin_memory = is_pin_memory_available()
@@ -603,34 +567,9 @@ class SamplingTensors:
567 dtype=torch.int,
568 pin_memory=pin_memory,
569 )
606 - sample_indices_t = torch.tensor(
607 - sample_indices,
608 - device="cpu",
609 - dtype=torch.long,
610 - pin_memory=pin_memory,
611 - )
612 - # need to transpose and make contiguous to
613 - # copy the tensor correctly.
614 - # [batch_size, n_seeds] -> [n_seeds, batch_size]
615 - sampling_seeds_t = torch.tensor(
616 - sampling_seeds,
617 - device="cpu",
618 - dtype=torch.long,
619 - pin_memory=pin_memory,
620 - ).t().contiguous()
621 -
570 # Because the memory is pinned, we can do non-blocking
571 # transfer to device.
572
625 - # How many seeds the sample operation itself will need.
626 - num_base_seeds = sampling_seeds_t.shape[0] - extra_seeds_to_generate
627 - sampling_seeds_gpu = sampling_seeds_t.to(device=device,
628 - non_blocking=True)
629 - extra_seeds_gpu = sampling_seeds_gpu[num_base_seeds:]
630 - if not extra_seeds_gpu.numel():
631 - extra_seeds_gpu = None
632 - sampling_seeds_gpu = sampling_seeds_gpu[:num_base_seeds]
633 -
573 return cls(
574 temperatures=temperatures_t.to(device=device, non_blocking=True),
575 top_ps=top_ps_t.to(device=device, non_blocking=True),
@@ -644,38 +583,4 @@ class SamplingTensors:
583 non_blocking=True),
584 prompt_tokens=prompt_t.to(device=device, non_blocking=True),
585 output_tokens=output_t.to(device=device, non_blocking=True),
647 - sampling_seeds=sampling_seeds_gpu,
648 - sample_indices=sample_indices_t.to(device=device,
649 - non_blocking=True),
650 - extra_seeds=extra_seeds_gpu,
586 )
652 -
653 - @staticmethod
654 - def _get_sequence_seeds(
655 - seed: int,
656 - *extra_entropy: int,
657 - seeds_to_generate: int,
658 - is_greedy: bool,
659 - ):
660 - """Get `seeds_to_generate` child seeds from `seed` and extra entropy."""
661 - if not is_greedy:
662 - if seed is None:
663 - randint_fn = random.randint
664 - else:
665 - generator = random.Random(str((seed, ) + extra_entropy))
666 - randint_fn = generator.randint
667 - lo, hi = torch.iinfo(torch.long).min, torch.iinfo(torch.long).max
668 - # If the user/random sets seed = 0 but request should
669 - # have sampling, we need to change it to something
670 - # else. We use a constant in that case.
671 - # This way we don't need to create and load a bool
672 - # matrix in the sampling kernel, which reduces CPU
673 - # overhead and latency.
674 - seq_seeds = [
675 - randint_fn(lo, hi) or _SEED_0_REPLACEMENT
676 - for _ in range(seeds_to_generate)
677 - ]
678 - else:
679 - # For the kernel, seed == 0 means greedy decoding.
680 - seq_seeds = [0] * seeds_to_generate
681 - return seq_seeds
vllm/triton_utils/sample.py deleted
-13
@@ -1,13 +0,0 @@
1 -import math
2 -
3 -# This is a hardcoded limit in Triton (max block size).
4 -MAX_TRITON_N_COLS = 131072
5 -
6 -
7 -def get_num_triton_sampler_splits(n_cols: int) -> int:
8 - """Get the number of splits to use for Triton sampling.
9 -
10 - Triton has a limit on the number of columns it can handle, so we need to
11 - split the tensor and call the kernel multiple times if it's too large.
12 - """
13 - return math.ceil(n_cols / MAX_TRITON_N_COLS)
vllm/utils.py
+14 -23
@@ -270,7 +270,7 @@ class LRUCache(Generic[T]):
270
271
272 class PyObjectCache:
273 - """Used to cache python objects to avoid object allocations
273 + """Used to cache python objects to avoid object allocations
274 across scheduler iterations.
275 """
276
@@ -289,7 +289,7 @@ class PyObjectCache:
289 self._obj_cache.append(self._obj_builder())
290
291 def get_object(self):
292 - """Returns a pre-allocated cached object. If there is not enough
292 + """Returns a pre-allocated cached object. If there is not enough
293 objects, then the cache size will double.
294 """
295 if self._index >= len(self._obj_cache):
@@ -837,15 +837,6 @@ def async_tensor_h2d(
837 return t.to(device=target_device, non_blocking=True)
838
839
840 -def maybe_expand_dim(tensor: torch.Tensor,
841 - target_dims: int,
842 - size: int = 1) -> torch.Tensor:
843 - """Expand the tensor to the target_dims."""
844 - if tensor.ndim < target_dims:
845 - tensor = tensor.view(-1, *([size] * (target_dims - tensor.ndim)))
846 - return tensor
847 -
848 -
840 def get_dtype_size(dtype: torch.dtype) -> int:
841 """Get the size of the data type in bytes."""
842 return torch.tensor([], dtype=dtype).element_size()
@@ -1070,7 +1061,7 @@ def _cuda_device_count_stateless(
1061 def cuda_device_count_stateless() -> int:
1062 """Get number of CUDA devices, caching based on the value of
1063 CUDA_VISIBLE_DEVICES at the time of call.
1073 -
1064 +
1065 This should be used instead of torch.cuda.device_count()
1066 unless CUDA_VISIBLE_DEVICES has already been set to the desired
1067 value."""
@@ -1136,10 +1127,10 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1127 def _pull_args_from_config(args: List[str]) -> List[str]:
1128 """Method to pull arguments specified in the config file
1129 into the command-line args variable.
1139 -
1140 - The arguments in config file will be inserted between
1130 +
1131 + The arguments in config file will be inserted between
1132 the argument list.
1142 -
1133 +
1134 example:
1135 ```yaml
1136 port: 12323
@@ -1150,21 +1141,21 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1141 --config config.yaml -tp 2
1142 $: args = [
1143 "serve,chat,complete",
1153 - "facebook/opt-12B",
1154 - '--config', 'config.yaml',
1144 + "facebook/opt-12B",
1145 + '--config', 'config.yaml',
1146 '-tp', '2'
1147 ]
1148 $: args = [
1149 "serve,chat,complete",
1159 - "facebook/opt-12B",
1160 - '--port', '12323',
1161 - '--tensor-parallel-size', '4',
1150 + "facebook/opt-12B",
1151 + '--port', '12323',
1152 + '--tensor-parallel-size', '4',
1153 '-tp', '2'
1154 ]
1155 ```
1156
1157 Please note how the config args are inserted after the sub command.
1167 - this way the order of priorities is maintained when these are args
1158 + this way the order of priorities is maintained when these are args
1159 parsed by super().
1160 """
1161 assert args.count(
@@ -1190,7 +1181,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1181
1182 @staticmethod
1183 def _load_config_file(file_path: str) -> List[str]:
1193 - """Loads a yaml file and returns the key value pairs as a
1184 + """Loads a yaml file and returns the key value pairs as a
1185 flattened list with argparse like pattern
1186 ```yaml
1187 port: 12323
@@ -1201,7 +1192,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1192 '--port': '12323',
1193 '--tensor-parallel-size': '4'
1194 ]
1204 -
1195 +
1196 """
1197
1198 extension: str = file_path.split('.')[-1]