A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[refactor] remove triton based sampler (#8524)
Simon Mo committed
Sep 16, 2024 at 20:04 UTC
546034b466bf11f12936791312981b9982850eb0
9 files changed
+75
-1095
tests/kernels/test_rand.py
deleted
-52
@@ -1,52 +0,0 @@
1
-import random
2
-
3
-import pytest
4
-import torch
5
-
6
-from vllm.model_executor.layers.ops.rand import seeded_uniform
7
-from vllm.model_executor.utils import set_random_seed
8
-
9
-
10
-@pytest.mark.parametrize("dtype",
11
- [torch.float32, torch.float16, torch.bfloat16])
12
-@pytest.mark.parametrize("use_3d", [True, False])
13
-def test_seeded_uniform(dtype: torch.dtype, use_3d: bool):
14
- device = "cuda"
15
- for seed in range(512):
16
- set_random_seed(seed)
17
- rows = random.randint(1, 512)
18
- cols = random.randint(1, 64000)
19
- if use_3d:
20
- third_dim = random.randint(2, 10)
21
- dims = [rows, third_dim, cols]
22
- else:
23
- dims = [rows, cols]
24
- seeds = torch.randint(torch.iinfo(torch.long).min,
25
- torch.iinfo(torch.long).max, (rows, ),
26
- device=device)
27
-
28
- # Test that the same seed produces the same output
29
- out = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
30
- out2 = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
31
- torch.testing.assert_close(out, out2)
32
- # del to save memory
33
- del out2
34
-
35
- out3 = seeded_uniform(*dims, seeds=seeds, dtype=dtype, device=device)
36
- torch.testing.assert_close(out, out3)
37
- # del to save memory
38
- del out3
39
-
40
- # Initialize out tensor with garbage to ensure that it is overwritten
41
- out_with_tensor = seeded_uniform(
42
- *dims,
43
- out=torch.full(
44
- (*dims, ),
45
- -1,
46
- dtype=dtype,
47
- device=device,
48
- ),
49
- seeds=seeds,
50
- dtype=dtype,
51
- )
52
- torch.testing.assert_close(out, out_with_tensor)
tests/kernels/test_sampler.py
deleted
-209
@@ -1,209 +0,0 @@
1
-import gc
2
-from unittest.mock import patch
3
-
4
-import pytest
5
-import torch
6
-import triton
7
-import triton.language as tl
8
-
9
-from vllm.model_executor.layers.ops.sample import (_sample_triton,
10
- _uniform_to_exponential,
11
- sample)
12
-from vllm.model_executor.sampling_metadata import SamplingTensors
13
-from vllm.model_executor.utils import set_random_seed
14
-from vllm.triton_utils.libentry import LibEntry
15
-from vllm.triton_utils.sample import (MAX_TRITON_N_COLS,
16
- get_num_triton_sampler_splits)
17
-
18
-SINGLE_SPLIT_VOCAB_SIZE = 32000 # llama/mistral/mixtral vocab size
19
-MULTI_SPLIT_VOCAB_SIZE = MAX_TRITON_N_COLS + 100
20
-
21
-
22
-@pytest.fixture(autouse=True)
23
-def _cleanup():
24
- yield
25
- gc.collect()
26
- torch.cuda.empty_cache()
27
-
28
-
29
-@triton.jit
30
-def _uniform_to_exponential_kernel(input, output, n: tl.constexpr):
31
- idx = tl.arange(0, n)
32
- x = tl.load(input + idx)
33
- y = _uniform_to_exponential(x)
34
- tl.store(output + idx, y)
35
-
36
-
37
-def test_uniform_to_exponential():
38
- """Test that we can convert uniform to exponential without div by 0."""
39
- input = torch.tensor([0.0, 1.0 - torch.finfo(torch.float32).eps],
40
- dtype=torch.float32,
41
- device="cuda")
42
- output = torch.zeros(input.shape, dtype=torch.float32, device="cuda")
43
- _uniform_to_exponential_kernel[(1, )](input, output, 2)
44
- assert torch.all(torch.isfinite(output))
45
- assert torch.all(output > 0)
46
- assert torch.all(torch.isfinite(torch.full_like(output, 1.0) / output))
47
-
48
-
49
-@pytest.mark.parametrize("random_sampling", [True, False, "mixed"])
50
-@pytest.mark.parametrize("max_best_of", [1, 2, 3, 4, 5])
51
-@pytest.mark.parametrize("modify_greedy_probs", [True, False])
52
-@pytest.mark.parametrize("seed", [1337])
53
-@pytest.mark.parametrize("vocab_size",
54
- [SINGLE_SPLIT_VOCAB_SIZE, MULTI_SPLIT_VOCAB_SIZE])
55
-@pytest.mark.parametrize("save_logprobs", [True, False])
56
-def test_sample_decoding_only(random_sampling, max_best_of,
57
- modify_greedy_probs, seed, vocab_size,
58
- save_logprobs):
59
- set_random_seed(seed)
60
- bs = 8
61
- probs = torch.zeros((bs, vocab_size), dtype=torch.float32, device="cuda")
62
- for i in range(bs):
63
- probs[i, i * (vocab_size // bs)] = 1.0
64
- logprobs = torch.rand_like(probs)
65
- sample_indices = torch.arange(bs, dtype=torch.long, device="cuda")
66
- n_splits = get_num_triton_sampler_splits(probs.shape[1])
67
- if random_sampling == "mixed":
68
- random_sampling_mask = (torch.rand(
69
- (1, bs), device="cuda") < 0.5).expand(n_splits, bs)
70
- elif random_sampling:
71
- random_sampling_mask = torch.ones((n_splits, bs),
72
- dtype=torch.bool,
73
- device="cuda")
74
- else:
75
- random_sampling_mask = torch.zeros((n_splits, bs),
76
- dtype=torch.bool,
77
- device="cuda")
78
-
79
- seeds = torch.randint(1,
80
- torch.iinfo(torch.long).max, (n_splits, bs),
81
- device="cuda").mul_(random_sampling_mask)
82
- #The current _sample_triton does not utilize the
83
- # libentry decoration. The purpose of adding this patch is to test
84
- # the correctness of libentry.
85
- with patch("vllm.model_executor.layers.ops.sample._sample_triton",
86
- LibEntry(_sample_triton)):
87
- sampled_tokens, sampled_logprobs, sampled_modified_probs = sample(
88
- probs=probs,
89
- logprobs=logprobs,
90
- sample_indices=sample_indices,
91
- seeds=seeds,
92
- max_best_of=max_best_of,
93
- modify_greedy_probs=modify_greedy_probs,
94
- save_logprobs=save_logprobs,
95
- _save_modified_probs=True)
96
- assert sampled_tokens.shape == (bs, max_best_of)
97
- for i in range(bs):
98
- assert torch.all(sampled_tokens[i] == i * (vocab_size // bs))
99
- request_uses_random_sampling = random_sampling_mask[0, i]
100
- if modify_greedy_probs and not request_uses_random_sampling:
101
- # If we are modifying greedy probs and the request is greedy,
102
- # we want to make sure the probs tensor is modified in place
103
- torch.testing.assert_close(
104
- probs[i][sampled_tokens[i]],
105
- torch.full_like(probs[i][sampled_tokens[i]], 1.0))
106
- assert torch.sum(probs[i]) == 1.0
107
- torch.testing.assert_close(
108
- sampled_modified_probs[i][0],
109
- torch.full_like(sampled_modified_probs[i][0], 1.0))
110
- elif request_uses_random_sampling:
111
- # If the request is random, we want to make sure
112
- # sampled_modified_probs tensor has noise added
113
- # (and thus is different from probs tensor)
114
- assert not torch.allclose(sampled_modified_probs[i][0],
115
- probs[i][sampled_tokens[i]])
116
- elif not request_uses_random_sampling:
117
- # If the request is greedy and we are not modifying greedy probs,
118
- # we want to make sure sampled_modified_probs tensor is the same as
119
- # the probs tensor.
120
- torch.testing.assert_close(sampled_modified_probs[i],
121
- probs[i][sampled_tokens[i]])
122
-
123
- if save_logprobs:
124
- assert sampled_logprobs.shape == (bs, max_best_of)
125
- for i in range(bs):
126
- for best_of in range(max_best_of):
127
- assert torch.all(sampled_logprobs[i] == logprobs[i][
128
- sampled_tokens[i, best_of]])
129
- else:
130
- assert sampled_logprobs is None
131
-
132
-
133
-@pytest.mark.parametrize("random_sampling", [True, False, "mixed"])
134
-@pytest.mark.parametrize("max_best_of", [1, 2, 3, 4, 5])
135
-@pytest.mark.parametrize("modify_greedy_probs", [True, False])
136
-@pytest.mark.parametrize("seed", [1337])
137
-@pytest.mark.parametrize("vocab_size",
138
- [SINGLE_SPLIT_VOCAB_SIZE, MULTI_SPLIT_VOCAB_SIZE])
139
-def test_sample_prompt_logprobs(random_sampling, max_best_of,
140
- modify_greedy_probs, seed, vocab_size):
141
-
142
- set_random_seed(seed)
143
- prompt_sizes = [16, 32, 64, 128] * 2
144
- samples = 8
145
- bs = samples + sum(prompt_sizes)
146
- probs = torch.zeros((bs, vocab_size), dtype=torch.float32, device="cuda")
147
- for i in range(bs):
148
- probs[i, i * (vocab_size // bs)] = 1.0
149
- logprobs = torch.rand_like(probs)
150
- sample_indices = torch.tensor(prompt_sizes,
151
- dtype=torch.long,
152
- device="cuda").cumsum_(0)
153
- n_splits = get_num_triton_sampler_splits(probs.shape[1])
154
- if random_sampling == "mixed":
155
- random_sampling_mask = torch.rand(
156
- (n_splits, samples), device="cuda") < 0.5
157
- elif random_sampling:
158
- random_sampling_mask = torch.ones((n_splits, samples),
159
- dtype=torch.bool,
160
- device="cuda")
161
- else:
162
- random_sampling_mask = torch.zeros((n_splits, samples),
163
- dtype=torch.bool,
164
- device="cuda")
165
-
166
- seeds = torch.randint(1,
167
- torch.iinfo(torch.long).max, (n_splits, samples),
168
- device="cuda").mul_(random_sampling_mask)
169
- #ditto
170
- with patch("vllm.model_executor.layers.ops.sample._sample_triton",
171
- LibEntry(_sample_triton)):
172
- sampled_tokens, sampled_logprobs, _ = sample(
173
- probs=probs,
174
- logprobs=logprobs,
175
- sample_indices=sample_indices,
176
- seeds=seeds,
177
- max_best_of=max_best_of,
178
- modify_greedy_probs=modify_greedy_probs,
179
- save_logprobs=True)
180
- assert sampled_tokens.shape == (samples, max_best_of)
181
- assert sampled_logprobs.shape == (samples, max_best_of)
182
- for i, t in enumerate(sample_indices):
183
- assert torch.all(sampled_tokens[i] == t * (vocab_size // bs))
184
- for best_of in range(max_best_of):
185
- assert torch.all(sampled_logprobs[i] == logprobs[sample_indices[i]]
186
- [sampled_tokens[i, best_of]])
187
-
188
-
189
-@pytest.mark.parametrize("seed", list(range(16)))
190
-def test_get_sequence_seeds(seed):
191
- """Ensure that we get a different child seed from base
192
- seed + extra entropy"""
193
- starting_seed = seed
194
- seq_seed = None
195
- extra_entropy = 1
196
- for i in range(512):
197
- new_seq_seed = SamplingTensors._get_sequence_seeds(starting_seed,
198
- i,
199
- seeds_to_generate=1,
200
- is_greedy=False)[0]
201
- new_seq_seed_extra_entropy = SamplingTensors._get_sequence_seeds(
202
- starting_seed,
203
- i,
204
- extra_entropy,
205
- seeds_to_generate=1,
206
- is_greedy=False)[0]
207
- assert new_seq_seed_extra_entropy != new_seq_seed
208
- assert seq_seed != new_seq_seed
209
- seq_seed = new_seq_seed
vllm/model_executor/layers/ops/__init__.py
vllm/model_executor/layers/ops/rand.py
deleted
-157
@@ -1,157 +0,0 @@
1
-from typing import Optional, Union
2
-
3
-import torch
4
-import triton
5
-import triton.language as tl
6
-
7
-
8
-def seeded_uniform(
9
- *size,
10
- seeds: torch.Tensor,
11
- out: Optional[torch.Tensor] = None,
12
- dtype: Optional[torch.dtype] = None,
13
- device: Optional[Union[torch.device, str]] = None,
14
- pin_memory: Optional[bool] = False,
15
-) -> torch.Tensor:
16
- """Similar to torch.rand, but allows for seeds to be set per row.
17
-
18
- seeds must be a 1d tensor. The output tensor may be 1d, 2d, or 3d.
19
- If it is 3d, the additional seeds needed will be derived automatically
20
- in a deterministic fashion:
21
- [
22
- row 0: [columns_with_seed_0], [columns_with_seed0^1], ...
23
- ]
24
- """
25
- n_dims = len(size)
26
-
27
- if n_dims > 3:
28
- raise ValueError("seeded_uniform only supports up to 3D tensors")
29
-
30
- if out is None:
31
- out = torch.empty(*size,
32
- dtype=dtype,
33
- device=device,
34
- pin_memory=pin_memory)
35
- elif out.shape != size:
36
- raise ValueError("shape of out and size must be the same")
37
-
38
- if n_dims == 3:
39
- n_rows, n_3d, n_cols = out.shape
40
- stride_row = out.stride(0)
41
- stride_3d = out.stride(1)
42
- elif n_dims == 2:
43
- n_rows, n_cols = out.shape
44
- n_3d = 1
45
- stride_row = out.stride(0)
46
- stride_3d = 1
47
- else:
48
- n_cols = out.shape[0]
49
- n_rows = 1
50
- n_3d = 1
51
- stride_row = 1
52
- stride_3d = 1
53
-
54
- if seeds.ndim != 1:
55
- raise ValueError("seeds must be a 1D tensor")
56
-
57
- if seeds.numel() != n_rows:
58
- raise ValueError(
59
- "seeds must have the same number of elements as out has rows")
60
-
61
- # The philox PRNG Triton uses generates 4 random numbers at once.
62
- # Therefore, the most efficient use of it is to divide the
63
- # block size by 4, and then save the generated random numbers to
64
- # each of the 4 slices of the tensor.
65
- full_block_size = triton.next_power_of_2(n_cols)
66
- philox_block_size = max(full_block_size // 4, 1)
67
- n_slices = full_block_size // philox_block_size
68
- num_warps = 4
69
- # Manual tuning. This seems to give best performance on A100 for
70
- # simple kernels like this.
71
- if philox_block_size >= 8192:
72
- num_warps = 32
73
- elif philox_block_size >= 4096:
74
- num_warps = 16
75
- elif philox_block_size >= 2048:
76
- num_warps = 8
77
-
78
- _seeded_uniform_triton[(n_rows, n_3d)](
79
- out,
80
- seeds,
81
- stride_row,
82
- stride_3d,
83
- seeds.stride(0),
84
- n_rows,
85
- n_3d,
86
- n_cols,
87
- n_slices=n_slices,
88
- num_warps=num_warps,
89
- block_size=philox_block_size,
90
- )
91
- return out
92
-
93
-
94
-@triton.jit
95
-def _seeded_uniform_triton(
96
- out_ptr: torch.Tensor,
97
- seed_ptr: torch.Tensor,
98
- out_row_stride: int,
99
- out_3d_stride: int,
100
- seed_row_stride: int,
101
- n_rows: int,
102
- n_3d: int,
103
- n_cols: int,
104
- n_slices: tl.constexpr,
105
- block_size: tl.constexpr,
106
-):
107
- """
108
- Generate a random float32 number in [0, 1) for each element in the output
109
- tensor. The random numbers in a row generated using the seed for that row.
110
-
111
- Args:
112
- out_ptr: The output tensor.
113
- seed_ptr: The per-row seeds to use for random number generation.
114
- out_row_stride: The stride between rows of the output tensor.
115
- out_3d_stride: The stride between 3D slices of the output tensor.
116
- seed_row_stride: The stride between rows of the seed tensor.
117
- n_rows: The number of rows in the output tensor.
118
- n_3d: The size of second dimension of the output tensor,
119
- if output tensor is 3D.
120
- n_cols: The number of columns in the output tensor.
121
- n_slices: The number of philox outputs to use.
122
- """
123
- tl.static_assert(n_slices > 0 and n_slices <= 4, "0 < n_slices <= 4")
124
-
125
- # Get the row index.
126
- row_idx = tl.program_id(axis=0)
127
- three_d_idx = tl.program_id(axis=1)
128
-
129
- philox_offsets = tl.arange(0, block_size)
130
- # Get the seed for the current element.
131
- seed = tl.load(seed_ptr + row_idx * seed_row_stride)
132
- if three_d_idx > 0:
133
- seed ^= three_d_idx
134
- # Generate random numbers in [0, 1).
135
- out1, out2, out3, out4 = tl.rand4x(seed, philox_offsets)
136
-
137
- output_row_start_ptr = (out_ptr + row_idx * out_row_stride +
138
- three_d_idx * out_3d_stride)
139
- out1_offsets = philox_offsets
140
- tl.store(output_row_start_ptr + out1_offsets,
141
- out1,
142
- mask=out1_offsets < n_cols)
143
- if n_slices > 1:
144
- out2_offsets = tl.arange(block_size, block_size * 2)
145
- tl.store(output_row_start_ptr + out2_offsets,
146
- out2,
147
- mask=out2_offsets < n_cols)
148
- if n_slices > 2:
149
- out3_offsets = tl.arange(block_size * 2, block_size * 3)
150
- tl.store(output_row_start_ptr + out3_offsets,
151
- out3,
152
- mask=out3_offsets < n_cols)
153
- if n_slices > 3:
154
- out4_offsets = tl.arange(block_size * 3, block_size * 4)
155
- tl.store(output_row_start_ptr + out4_offsets,
156
- out4,
157
- mask=out4_offsets < n_cols)
vllm/model_executor/layers/ops/sample.py
deleted
-394
@@ -1,394 +0,0 @@
1
-from typing import Optional, Tuple
2
-
3
-import torch
4
-import triton
5
-import triton.language as tl
6
-
7
-from vllm.model_executor.layers.ops.rand import seeded_uniform
8
-from vllm.triton_utils.sample import get_num_triton_sampler_splits
9
-
10
-_EPS: tl.constexpr = 1e-6
11
-
12
-
13
-def _multi_split_sample(
14
- probs: torch.Tensor,
15
- seeds: torch.Tensor,
16
- n_splits: int,
17
- sampled_tokens_size: Tuple[int, int],
18
- sampled_logprobs_size: Tuple[int, int],
19
- sample_indices: torch.Tensor,
20
- logprobs: torch.Tensor,
21
- *,
22
- modify_greedy_probs: bool = False,
23
- save_logprobs: bool = False,
24
-):
25
- """Sample tokens where vocab size is split into multiple parts
26
- (too large for Triton otherwise)."""
27
- assert seeds.ndim == 2 and seeds.shape[0] == n_splits
28
- split_probs = probs.tensor_split(n_splits, 1)
29
- split_logprobs = logprobs.tensor_split(n_splits, 1)
30
- sampled_tokens_tmp = [
31
- torch.empty(sampled_tokens_size, dtype=torch.long, device=probs.device)
32
- for _ in range(n_splits)
33
- ]
34
- sampled_logprobs_tmp = [
35
- torch.empty(sampled_logprobs_size,
36
- dtype=probs.dtype,
37
- device=probs.device) for _ in range(n_splits)
38
- ]
39
- # We are purposefuly using sampled_tokens_size as we need to always
40
- # save modified probs in this case.
41
- sampled_modified_probs_tmp = [
42
- torch.empty(sampled_tokens_size,
43
- dtype=probs.dtype,
44
- device=probs.device) for _ in range(n_splits)
45
- ]
46
- for i in range(n_splits):
47
- n_samples = sample_indices.shape[0]
48
- n_cols = split_probs[i].shape[1]
49
- n_best = sampled_tokens_tmp[i].shape[1]
50
- uniform_noise = seeded_uniform(n_samples,
51
- n_best,
52
- n_cols,
53
- seeds=seeds[i].flatten(),
54
- device=split_probs[i].device,
55
- dtype=split_probs[i].dtype)
56
- # TODO(yard1): See if we can remove the contiguous() calls.
57
- # Will need kernel support.
58
- _sample(
59
- split_probs[i].contiguous(),
60
- split_logprobs[i].contiguous(),
61
- sample_indices,
62
- sampled_tokens_tmp[i],
63
- sampled_logprobs_tmp[i],
64
- sampled_modified_probs_tmp[i],
65
- seeds[i],
66
- uniform_noise,
67
- modify_greedy_probs=False,
68
- save_logprobs=save_logprobs,
69
- save_modified_probs=True,
70
- )
71
- if i > 0:
72
- # Add offset to sampled tokens
73
- sampled_tokens_tmp[i].add_(i * split_probs[i - 1].shape[1])
74
- sampled_tokens = torch.stack(sampled_tokens_tmp)
75
- sampled_modified_probs = torch.stack(sampled_modified_probs_tmp)
76
- # Reduce the results from the splits.
77
- sampled_modified_probs, indices = torch.max(sampled_modified_probs,
78
- dim=0,
79
- keepdim=True)
80
- sampled_tokens = sampled_tokens.gather(0, indices).squeeze(0)
81
- if save_logprobs:
82
- sampled_logprobs = torch.stack(sampled_logprobs_tmp)
83
- sampled_logprobs = sampled_logprobs.gather(0, indices).squeeze(0)
84
- else:
85
- sampled_logprobs = None
86
- sampled_modified_probs = sampled_modified_probs.squeeze(0)
87
-
88
- if modify_greedy_probs:
89
- # We need to modify the greedy probs for the sampled tokens.
90
- # We can't do this in the kernel as we need to know the
91
- # sampled tokens.
92
- probs.fill_(0.0)
93
- probs.scatter_(1, sampled_tokens, 1.0)
94
-
95
- return (sampled_tokens, sampled_logprobs, sampled_modified_probs)
96
-
97
-
98
-def sample(
99
- probs: torch.Tensor,
100
- seeds: torch.Tensor,
101
- *,
102
- max_best_of: int = 1,
103
- sample_indices: Optional[torch.Tensor] = None,
104
- logprobs: Optional[torch.Tensor] = None,
105
- modify_greedy_probs: bool = False,
106
- save_logprobs: bool = False,
107
- _save_modified_probs: bool = False, # pylint: disable=invalid-name
108
-) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]:
109
- """Sample tokens from probs. with per-sequence seeds.
110
-
111
- Can sample from a subset of sequences through sample_indices.
112
-
113
- Args:
114
- probs: Probabilities to sample from.
115
- shape = [batch_size, vocab_size]
116
- seeds: Per-sequence seed values.
117
- shape = [n, math.ceil(vocab_size / MAX_TRITON_N_COLS)]
118
- max_best_of: Number of samples to generate per sequence.
119
- Sequence seed will be incremented by 1 each time.
120
- sample_indices: Indices of sequences to sample from.
121
- If not provided, will sample from all sequences.
122
- shape = [n]
123
- logprobs: Log-probabilities of the sampled tokens.
124
- Only used for saving the logprobs if save_logprobs is True.
125
- shape = [batch_size, vocab_size]
126
- modify_greedy_probs: Whether to modify the greedy probabilities
127
- for speculative sampling (sampled token = 1.0,
128
- everything else = 0.0).
129
- save_logprobs: Whether to save the log-probabilities of the
130
- sampled tokens to a tensor.
131
- _save_modified_probs: Whether to save the modified probabilities
132
- (including gumbel noise) of the sampled tokens to a tensor.
133
- DOES NOT include the modification done by modify_greedy_probs
134
- (because we want to use the unmodified probs to pick the best
135
- split in case of multi-split sampling).
136
- This is exposed only for testing.
137
-
138
- Returns:
139
- sampled_tokens: shape = [n, max_best_of]
140
- sampled_logprobs: shape = [n, max_best_of] if save_logprobs else None
141
- sampled_modified_probs: shape = [n, max_best_of]
142
- if save_modified_probs else None
143
- """
144
- if sample_indices is None:
145
- sample_indices = torch.arange(0, probs.shape[0], device=probs.device)
146
-
147
- sampled_tokens_size = (sample_indices.size(0), max_best_of)
148
- if save_logprobs:
149
- if logprobs is None:
150
- raise ValueError(
151
- "logprobs tensor must be provided if save_logprobs is True")
152
- sampled_logprobs_size = sampled_tokens_size
153
- else:
154
- # Empty tensors to invoke the kernel
155
- sampled_logprobs_size = (0, 0)
156
- logprobs = probs
157
-
158
- assert logprobs is not None
159
- if _save_modified_probs:
160
- sampled_modified_probs_size = sampled_tokens_size
161
- else:
162
- # Empty tensors to invoke the kernel
163
- sampled_modified_probs_size = (0, 0)
164
-
165
- # If the number of columns in probs is too large for Triton to handle,
166
- # we split the tensor and sample from each split separately, and then
167
- # do an argmax+gather to combine the results.
168
- n_splits = get_num_triton_sampler_splits(probs.shape[1])
169
- if n_splits > 1:
170
- (sampled_tokens, sampled_logprobs,
171
- sampled_modified_probs) = _multi_split_sample(
172
- probs,
173
- seeds,
174
- n_splits,
175
- sampled_tokens_size,
176
- sampled_logprobs_size,
177
- sample_indices,
178
- logprobs=logprobs,
179
- modify_greedy_probs=modify_greedy_probs,
180
- save_logprobs=save_logprobs)
181
- else:
182
- sampled_tokens = torch.empty(sampled_tokens_size,
183
- dtype=torch.long,
184
- device=probs.device)
185
- sampled_logprobs = torch.empty(sampled_logprobs_size,
186
- dtype=probs.dtype,
187
- device=probs.device)
188
- sampled_modified_probs = torch.empty(sampled_modified_probs_size,
189
- dtype=probs.dtype,
190
- device=probs.device)
191
- n_samples = sample_indices.shape[0]
192
- n_cols = probs.shape[1]
193
- uniform_noise = seeded_uniform(n_samples,
194
- max_best_of,
195
- n_cols,
196
- seeds=seeds.flatten(),
197
- device=probs.device,
198
- dtype=probs.dtype)
199
-
200
- _sample(
201
- probs,
202
- logprobs,
203
- sample_indices,
204
- sampled_tokens,
205
- sampled_logprobs,
206
- sampled_modified_probs,
207
- seeds,
208
- uniform_noise,
209
- modify_greedy_probs=modify_greedy_probs,
210
- save_logprobs=save_logprobs,
211
- save_modified_probs=_save_modified_probs,
212
- )
213
- return (sampled_tokens, sampled_logprobs if save_logprobs else None,
214
- sampled_modified_probs if _save_modified_probs else None)
215
-
216
-
217
-def _sample(probs: torch.Tensor,
218
- logprobs: torch.Tensor,
219
- sample_indices: torch.Tensor,
220
- output_samples: torch.Tensor,
221
- output_logprobs: torch.Tensor,
222
- output_modified_probs: torch.Tensor,
223
- seeds: torch.Tensor,
224
- uniform_noise: torch.Tensor,
225
- *,
226
- modify_greedy_probs: bool = False,
227
- save_logprobs: bool = True,
228
- save_modified_probs: bool = False) -> torch.Tensor:
229
- """Sample tokens from probs.
230
-
231
- Args:
232
- probs [batch_size, vocab_size]: probs to sample from.
233
- logprobs [batch_size, vocab_size]: logprobs (used when
234
- save_logprobsis True).
235
- sample_indices [n]: Indices of the samples to use for each row of probs.
236
- output_samples [n, n_best]: Output tensor to store samples in.
237
- output_logprobs [n, n_best]: Output tensor to store logprobs in.
238
- output_modified_probs [n, n_best]: Output tensor to store
239
- probs of chosen tokens in (modified with noise).
240
- seeds [n]: Seeds to use for sampling. If the seed is 0, we use
241
- greedy sampling. Note this is ONLY used for determining
242
- whether to use random sampling or not. The actual random
243
- noise should be passed as uniform_noise.
244
- uniform_noise [batch_size, n_best, vocab_size]: Uniform
245
- noise to use for random sampling (will be converted
246
- to exponential gumbel noise by the kernel).
247
- modify_greedy_probs: If True, we modify the probs tensor in-place
248
- to encode the sampling method used for each row. This is used
249
- in speculative decoding. Only applies in greedy decoding.
250
- save_logprobs: If True, we save the logprobs of the sampled tokens
251
- in the output_logprobs tensor.
252
- save_modified_probs: If True, we save the modified probs (with noise)
253
- of the sampled tokens in the output_modified_probs tensor.
254
- DOES NOT include the modification done by modify_greedy_probs
255
- (because we want to use the unmodified probs to pick the best
256
- split in case of multi-split sampling).
257
- """
258
- n_samples = sample_indices.shape[0]
259
- n_cols = probs.shape[1]
260
- n_best = output_samples.shape[1] if len(output_samples.shape) > 1 else 1
261
-
262
- # The block size is the smallest power of two greater than the number of
263
- # columns in probs
264
- block_size = triton.next_power_of_2(n_cols)
265
- num_warps = 4
266
- # Manual tuning. This seems to give best performance on A100 for
267
- # simple kernels like this.
268
- if block_size >= 8192:
269
- num_warps = 32
270
- elif block_size >= 4096:
271
- num_warps = 16
272
- elif block_size >= 2048:
273
- num_warps = 8
274
-
275
- # Enqueue kernel. The 1D launch grid is simple: we have one kernel
276
- # instance per row of the probs matrix
277
- _sample_triton[(n_samples, n_best)](
278
- sample_indices,
279
- output_samples,
280
- output_logprobs,
281
- output_modified_probs,
282
- probs,
283
- logprobs,
284
- seeds,
285
- uniform_noise,
286
- output_samples.stride(0),
287
- probs.stride(0),
288
- uniform_noise.stride(0),
289
- uniform_noise.stride(1) if n_best > 1 else 1,
290
- n_samples,
291
- n_cols,
292
- n_best,
293
- num_warps=num_warps,
294
- block_size=block_size,
295
- modify_greedy_probs=modify_greedy_probs,
296
- save_logprobs=save_logprobs,
297
- save_modified_probs=save_modified_probs,
298
- )
299
- return output_samples, output_logprobs, output_modified_probs
300
-
301
-
302
-@triton.jit
303
-def _uniform_to_exponential(uniform_noise):
304
- """Convert uniform samples to exponential samples."""
305
- # tl.rand returns values in [0, 1), so we clamp lower bound
306
- # to _EPS to avoid log(0) and thus division by 0 later
307
- lb = tl.full(uniform_noise.shape, _EPS, uniform_noise.dtype)
308
- uniform_noise = tl.maximum(uniform_noise, lb)
309
- # Use the inversion method to turn uniform samples
310
- # into exponential samples
311
- exponential_noise = -tl.log(uniform_noise)
312
- return exponential_noise
313
-
314
-
315
-@triton.jit
316
-def _sample_triton(
317
- sample_indices_ptr: torch.Tensor, output_ptr: torch.Tensor,
318
- output_logprobs_ptr: torch.Tensor,
319
- output_modified_probs_ptr: torch.Tensor, probs_ptr: torch.Tensor,
320
- logprobs_ptr: torch.Tensor, seeds_ptr: torch.Tensor,
321
- uniform_noise_ptr: torch.Tensor, output_row_stride: int,
322
- probs_row_stride: int, uniform_noise_row_stride: int,
323
- uniform_noise_best_stride: int, n_samples: int, n_cols: int,
324
- n_best: int, block_size: tl.constexpr,
325
- modify_greedy_probs: tl.constexpr, save_logprobs: tl.constexpr,
326
- save_modified_probs: tl.constexpr):
327
- # The rows are independent, so we parallelize across those
328
- sample_idx = tl.program_id(0)
329
- best_idx = tl.program_id(1)
330
-
331
- # Load the row index from DRAM
332
- row_idx = tl.load(sample_indices_ptr + sample_idx)
333
- seed = tl.load(seeds_ptr + sample_idx)
334
- uses_random_sampling = seed != 0
335
-
336
- # The stride represents how much we need to increase the
337
- # pointer to advance 1 row
338
- row_start_ptr = probs_ptr + row_idx * probs_row_stride
339
-
340
- # The block size is the next power of two greater than n_cols,
341
- # so we can fit each row in a single block
342
- col_offsets = tl.arange(0, block_size)
343
-
344
- # Load the row into SRAM, using a mask since block_size may be > than n_cols
345
- row = tl.load(row_start_ptr + col_offsets,
346
- mask=col_offsets < n_cols,
347
- other=float("-inf"))
348
-
349
- if uses_random_sampling:
350
- uniform_noise_start_ptr = (uniform_noise_ptr +
351
- sample_idx * uniform_noise_row_stride +
352
- best_idx * uniform_noise_best_stride)
353
- uniform_noise = tl.load(uniform_noise_start_ptr + col_offsets,
354
- mask=col_offsets < n_cols,
355
- other=0.5)
356
- exponential_noise = _uniform_to_exponential(uniform_noise)
357
- row /= exponential_noise
358
-
359
- sampled_value, sampled_token = tl.max(row, axis=0, return_indices=True)
360
- # clamp sampled token to n_cols - 1
361
- # this should not be necessary, but we do it
362
- # just in case
363
- if sampled_token >= n_cols:
364
- sampled_token = n_cols - 1
365
- # Write back output to DRAM
366
- output_row_start_ptr = (output_ptr + sample_idx * output_row_stride +
367
- best_idx)
368
- tl.store(output_row_start_ptr, sampled_token)
369
-
370
- if modify_greedy_probs: # noqa
371
- if not uses_random_sampling:
372
- # Set the probability of the sampled token to 1, all other
373
- # tokens to zero. This is used in speculative decoding where
374
- # the sampling method must be encoded within the sampled
375
- # probability distributions.
376
- row = tl.where(col_offsets == sampled_token, 1.0, 0.0)
377
- tl.store(row_start_ptr + col_offsets,
378
- row,
379
- mask=col_offsets < n_cols)
380
-
381
- if save_modified_probs:
382
- output_row_start_ptr = (output_modified_probs_ptr +
383
- sample_idx * output_row_stride + best_idx)
384
- tl.store(output_row_start_ptr, sampled_value)
385
-
386
- if save_logprobs:
387
- # Load the row into SRAM, using a mask since block_size
388
- # may be > than n_cols
389
- sampled_logprob = tl.load(logprobs_ptr + row_idx * probs_row_stride +
390
- sampled_token)
391
- # Write back output to DRAM
392
- output_row_start_ptr = (output_logprobs_ptr +
393
- sample_idx * output_row_stride + best_idx)
394
- tl.store(output_row_start_ptr, sampled_logprob)
vllm/model_executor/layers/sampler.py
+3
-94
@@ -10,12 +10,6 @@ import msgspec
10
import torch
11
import torch.nn as nn
12
13
-from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
14
-from vllm.triton_utils import HAS_TRITON
15
-
16
-if HAS_TRITON:
17
- from vllm.model_executor.layers.ops.sample import sample as sample_triton
18
-
13
import vllm.envs as envs
14
from vllm.model_executor.sampling_metadata import (SamplingMetadata,
15
SamplingTensors,
@@ -23,6 +17,7 @@ from vllm.model_executor.sampling_metadata import (SamplingMetadata,
17
from vllm.sampling_params import SamplingType
18
from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
19
PromptLogprobs, SampleLogprobs, SequenceOutput)
20
+from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
21
22
if envs.VLLM_USE_FLASHINFER_SAMPLER and find_spec("flashinfer"):
23
import flashinfer.sampling
@@ -740,7 +735,7 @@ def _sample_with_torch(
735
) -> SampleReturnType:
736
'''Torch-oriented _sample() implementation.
737
743
- Single-step scheduling:
738
+ Single-step scheduling:
739
* Perform GPU-side sampling computation
740
* Immediately Pythonize sampling result
741
@@ -777,7 +772,7 @@ def _sample_with_torch(
772
# Counterintiutively, having two loops here is actually faster.
773
# The first loop can run without waiting on GPU<->CPU sync.
774
for sampling_type in SamplingType:
780
- sample_indices = categorized_sample_indices[sampling_type][:, 0]
775
+ sample_indices = categorized_sample_indices[sampling_type]
776
num_tokens = len(sample_indices)
777
if num_tokens == 0:
778
continue
@@ -863,88 +858,6 @@ def _sample_with_torch(
858
)
859
860
866
-def _sample_with_triton_kernel(
867
- probs: torch.Tensor,
868
- logprobs: torch.Tensor,
869
- sampling_metadata: SamplingMetadata,
870
- sampling_tensors: SamplingTensors,
871
-) -> SampleResultType:
872
- categorized_seq_group_ids: Dict[SamplingType,
873
- List[int]] = {t: []
874
- for t in SamplingType}
875
- categorized_sample_indices = sampling_metadata.categorized_sample_indices
876
- for i, seq_group in enumerate(sampling_metadata.seq_groups):
877
- sampling_params = seq_group.sampling_params
878
- sampling_type = sampling_params.sampling_type
879
- categorized_seq_group_ids[sampling_type].append(i)
880
-
881
- sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
882
- sample_metadata: Dict[SamplingType,
883
- Tuple[List[int], List[SequenceGroupToSample],
884
- torch.Tensor, torch.Tensor]] = {}
885
- max_best_of_in_batch = 1
886
-
887
- # Counterintiutively, having two loops here is actually faster.
888
- # The first loop can run without waiting on GPU<->CPU sync.
889
- for sampling_type in SamplingType:
890
- sample_indices = categorized_sample_indices[sampling_type][:, 0]
891
- sampled_token_indices = categorized_sample_indices[sampling_type][:, 1]
892
- num_tokens = len(sample_indices)
893
- if num_tokens == 0:
894
- continue
895
- seq_group_id = categorized_seq_group_ids[sampling_type]
896
- seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_id]
897
- sample_metadata[sampling_type] = (seq_group_id, seq_groups,
898
- sample_indices,
899
- sampled_token_indices)
900
- if sampling_type in (SamplingType.GREEDY, SamplingType.RANDOM,
901
- SamplingType.RANDOM_SEED):
902
- for seq_group in seq_groups:
903
- if seq_group.is_prompt:
904
- sampling_params = seq_group.sampling_params
905
- max_best_of_in_batch = max(max_best_of_in_batch,
906
- sampling_params.best_of)
907
- elif sampling_type == SamplingType.BEAM:
908
- beam_search_logprobs = logprobs[sample_indices]
909
- else:
910
- raise ValueError(f"Unsupported sampling type: {sampling_type}")
911
-
912
- sampled_tokens, _, _ = sample_triton(
913
- probs=probs,
914
- seeds=sampling_tensors.sampling_seeds,
915
- max_best_of=max_best_of_in_batch,
916
- sample_indices=sampling_tensors.sample_indices,
917
- logprobs=logprobs,
918
- # don't save logprobs because we have logic for that below
919
- # TODO: use this instead of the CPU-based logic below
920
- save_logprobs=False,
921
- )
922
-
923
- # GPU<->CPU sync happens in the loop below.
924
-
925
- for sampling_type in SamplingType:
926
- if sampling_type not in sample_metadata:
927
- continue
928
- (seq_group_id, seq_groups, sample_indices,
929
- sampled_token_indices) = sample_metadata[sampling_type]
930
- if sampling_type == SamplingType.GREEDY:
931
- sample_results = _greedy_sample(
932
- seq_groups, sampled_tokens[sampled_token_indices][:, 0])
933
- elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
934
- sample_results = _random_sample(
935
- seq_groups, sampled_tokens[sampled_token_indices])
936
- elif sampling_type == SamplingType.BEAM:
937
- sample_results = _beam_search_sample(seq_groups,
938
- beam_search_logprobs)
939
- sample_results_dict.update(zip(seq_group_id, sample_results))
940
-
941
- sample_results = [
942
- sample_results_dict.get(i, ([], []))
943
- for i in range(len(sampling_metadata.seq_groups))
944
- ]
945
- return sample_results
946
-
947
-
861
def _sample(
862
probs: torch.Tensor,
863
logprobs: torch.Tensor,
@@ -974,10 +887,6 @@ def _sample(
887
modify_greedy_probs=modify_greedy_probs,
888
)
889
977
- # TODO: Enable once Triton kernel & associated code is faster.
978
- # return _sample_with_triton_kernel(probs, logprobs, sampling_metadata,
979
- # sampling_tensors)
980
-
890
891
def _get_ranks(x: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
892
"""
vllm/model_executor/sampling_metadata.py
+58
-153
@@ -1,4 +1,3 @@
1
-import random
1
from array import array
2
from dataclasses import dataclass
3
from typing import Dict, List, Optional, Tuple
@@ -8,15 +7,10 @@ import torch
7
from vllm.sampling_params import SamplingParams, SamplingType
8
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData,
9
SequenceGroupMetadata)
11
-from vllm.triton_utils.sample import get_num_triton_sampler_splits
10
from vllm.utils import (PyObjectCache, async_tensor_h2d,
13
- is_pin_memory_available, make_tensor_with_pad,
14
- maybe_expand_dim)
11
+ is_pin_memory_available, make_tensor_with_pad)
12
13
_SAMPLING_EPS = 1e-5
17
-_SEED_0_REPLACEMENT = 3403598558
18
-# Some triton sampler related code is guarded before it is ready.
19
-_USE_TRITON_SAMPLER = False
14
15
16
@dataclass
@@ -74,12 +68,12 @@ def gen_seq_group_to_sample_builder(num_seqs: int):
68
generator=None,
69
is_prompt=True,
70
prompt_logprob_indices=[],
77
- sample_indices=[])
71
+ sample_indices=[],
72
+ )
73
74
75
class SamplingMetadataCache:
81
- """Used to cache SamplingMetadata objects between scheduler iterations
82
- """
76
+ """Used to cache SamplingMetadata objects between scheduler iterations"""
77
78
def __init__(self):
79
self._seq_group_to_sample_cache: Dict[int, PyObjectCache] = {}
@@ -124,12 +118,12 @@ class SamplingMetadata:
118
The first tuple is [1, 2] (sampled index within original logit),
119
and the second tuple is [0, 1] (sampled index within pruned logit).
120
num_prompts: Number of prompt sequence groups in seq_groups.
127
- skip_sampler_cpu_output: Indicates if we want to skip the GPU=>CPU
121
+ skip_sampler_cpu_output: Indicates if we want to skip the GPU=>CPU
122
serialization of token outputs.
129
- reuse_sampling_tensors: Indicates if we want to reuse sampling
123
+ reuse_sampling_tensors: Indicates if we want to reuse sampling
124
tensors that are part of the sampler forward pass. Currently,
125
it is mainly used for multi-step decode.
132
-
126
+
127
"""
128
129
def __init__(
@@ -165,16 +159,19 @@ class SamplingMetadata:
159
num_prompts,
160
) = _prepare_seq_groups(seq_group_metadata_list, seq_lens, query_lens,
161
device, generators, cache)
168
- selected_token_indices = async_tensor_h2d(selected_token_indices,
169
- dtype=torch.long,
170
- target_device=device,
171
- pin_memory=pin_memory)
162
+ selected_token_indices = async_tensor_h2d(
163
+ selected_token_indices,
164
+ dtype=torch.long,
165
+ target_device=device,
166
+ pin_memory=pin_memory,
167
+ )
168
categorized_sample_indices = {
173
- t: maybe_expand_dim(
174
- async_tensor_h2d(seq_ids,
175
- dtype=torch.int,
176
- target_device=device,
177
- pin_memory=pin_memory), 2, 2)
169
+ t: async_tensor_h2d(
170
+ seq_ids,
171
+ dtype=torch.int,
172
+ target_device=device,
173
+ pin_memory=pin_memory,
174
+ )
175
for t, seq_ids in categorized_sample_indices.items()
176
}
177
@@ -201,8 +198,8 @@ def _prepare_seq_groups(
198
device: str,
199
generators: Optional[Dict[str, torch.Generator]] = None,
200
cache: Optional[SamplingMetadataCache] = None,
204
-) -> Tuple[List[SequenceGroupToSample], List[int], Dict[
205
- SamplingType, List[Tuple[int, int]]], int]:
201
+) -> Tuple[List[SequenceGroupToSample], List[int], Dict[SamplingType,
202
+ List[int]], int, ]:
203
"""Prepare sequence groups and indices for sampling.
204
205
Args:
@@ -233,16 +230,13 @@ def _prepare_seq_groups(
230
# Sampling type -> (
231
# indices to sample/prompt logprob within pruned output logits,
232
# indices to sample within pruned logits)
236
- categorized_sample_indices: Dict[SamplingType, List[Tuple[int, int]]] = {
233
+ categorized_sample_indices: Dict[SamplingType, List[int]] = {
234
t: []
235
for t in SamplingType
236
}
237
# Index of logits to compute logprob. Logits include both prompt logprob
238
# and sample logprob indices.
239
logit_idx = 0
243
- # Index to sample from a sample tensor. It is used by triton sample kernel.
244
- # See `_sample_with_triton_kernel` for more details.
245
- sample_idx = 0
240
# Total number of prompts from given sequence groups.
241
num_prompts = 0
242
@@ -264,10 +258,10 @@ def _prepare_seq_groups(
258
# If the current seq group is in decode stage, it is None.
259
seq_len: Optional[int] = None
260
query_len: Optional[int] = None
267
- prompt_logprob_indices: List[int] = \
268
- sample_obj.prompt_logprob_indices if cache is not None else []
269
- sample_indices: List[int] = \
270
- sample_obj.sample_indices if cache is not None else []
261
+ prompt_logprob_indices: List[int] = (sample_obj.prompt_logprob_indices
262
+ if cache is not None else [])
263
+ sample_indices: List[int] = (sample_obj.sample_indices
264
+ if cache is not None else [])
265
do_sample = seq_group_metadata.do_sample
266
267
if seq_group_metadata.is_prompt:
@@ -333,11 +327,8 @@ def _prepare_seq_groups(
327
if do_sample:
328
sample_indices.extend(range(logit_idx, logit_idx + sample_len))
329
categorized_sample_indices[sampling_params.sampling_type].extend(
336
- list(
337
- zip(range(logit_idx, logit_idx + sample_len),
338
- range(sample_idx, sample_idx + sample_len))))
330
+ list(range(logit_idx, logit_idx + sample_len)))
331
logit_idx += sample_len
340
- sample_idx += sample_len
332
333
if cache is not None:
334
sample_obj.sampling_params = sampling_params
@@ -356,7 +347,8 @@ def _prepare_seq_groups(
347
generator=generator,
348
is_prompt=is_prompt,
349
prompt_logprob_indices=list(prompt_logprob_indices),
359
- sample_indices=list(sample_indices))
350
+ sample_indices=list(sample_indices),
351
+ )
352
353
seq_groups.append(sample_obj)
354
@@ -378,9 +370,6 @@ class SamplingTensors:
370
presence_penalties: torch.Tensor
371
frequency_penalties: torch.Tensor
372
repetition_penalties: torch.Tensor
381
- sampling_seeds: torch.Tensor
382
- sample_indices: torch.Tensor
383
- extra_seeds: Optional[torch.Tensor]
373
prompt_tokens: torch.Tensor
374
output_tokens: torch.Tensor
375
@@ -391,15 +380,7 @@ class SamplingTensors:
380
vocab_size: int,
381
device: torch.device,
382
dtype: torch.dtype,
394
- *,
395
- extra_seeds_to_generate: int = 0,
396
- extra_entropy: Optional[Tuple[int, ...]] = None
383
) -> Tuple["SamplingTensors", bool, bool, bool]:
398
- """
399
- extra_seeds_to_generate: extra seeds to generate using the
400
- user-defined seed for each sequence.
401
- extra_entropy: extra entropy to use when generating seeds.
402
- """
384
prompt_tokens: List[array] = []
385
output_tokens: List[array] = []
386
top_ks: List[int] = []
@@ -409,19 +390,10 @@ class SamplingTensors:
390
presence_penalties: List[float] = []
391
frequency_penalties: List[float] = []
392
repetition_penalties: List[float] = []
412
- sampling_seeds: List[int] = []
413
- sample_indices: List[int] = []
393
do_penalties = False
394
do_top_p_top_k = False
395
do_min_p = False
396
418
- if _USE_TRITON_SAMPLER:
419
- prompt_best_of: List[int] = []
420
-
421
- # We need one base seed per Triton slice.
422
- seeds_to_generate = (extra_seeds_to_generate +
423
- get_num_triton_sampler_splits(vocab_size))
424
-
397
assert sampling_metadata.seq_groups is not None
398
for seq_group in sampling_metadata.seq_groups:
399
seq_ids = seq_group.seq_ids
@@ -452,7 +424,7 @@ class SamplingTensors:
424
do_penalties = True
425
426
is_prompt = seq_group.is_prompt
455
- if (is_prompt and sampling_params.prompt_logprobs is not None):
427
+ if is_prompt and sampling_params.prompt_logprobs is not None:
428
# For tokens in the prompt that we only need to get
429
# their logprobs
430
query_len = seq_group.query_len
@@ -477,28 +449,6 @@ class SamplingTensors:
449
frequency_penalties += [f] * len(seq_ids)
450
repetition_penalties += [r] * len(seq_ids)
451
480
- if _USE_TRITON_SAMPLER:
481
- if is_prompt:
482
- prompt_best_of.append(sampling_params.best_of)
483
- query_len = seq_group.query_len
484
- assert query_len is not None
485
-
486
- seed = sampling_params.seed
487
- is_greedy = sampling_params.sampling_type == SamplingType.GREEDY
488
-
489
- for seq_id in seq_ids:
490
- seq_data = seq_group.seq_data[seq_id]
491
- extra_entropy = extra_entropy or ()
492
- seq_seeds = cls._get_sequence_seeds(
493
- seed,
494
- seq_data.get_len(),
495
- *extra_entropy,
496
- seq_id,
497
- seeds_to_generate=seeds_to_generate,
498
- is_greedy=is_greedy)
499
- sampling_seeds.append(seq_seeds)
500
- sample_indices.extend(seq_group.sample_indices)
501
-
452
if do_penalties:
453
for seq_group in sampling_metadata.seq_groups:
454
seq_ids = seq_group.seq_ids
@@ -518,23 +468,37 @@ class SamplingTensors:
468
output_tokens.append(seq_data.output_token_ids_array)
469
470
sampling_tensors = SamplingTensors.from_lists(
521
- temperatures, top_ps, top_ks, min_ps, presence_penalties,
522
- frequency_penalties, repetition_penalties, sampling_seeds,
523
- sample_indices, prompt_tokens, output_tokens, vocab_size,
524
- extra_seeds_to_generate, device, dtype)
471
+ temperatures,
472
+ top_ps,
473
+ top_ks,
474
+ min_ps,
475
+ presence_penalties,
476
+ frequency_penalties,
477
+ repetition_penalties,
478
+ prompt_tokens,
479
+ output_tokens,
480
+ vocab_size,
481
+ device,
482
+ dtype,
483
+ )
484
return (sampling_tensors, do_penalties, do_top_p_top_k, do_min_p)
485
486
@classmethod
528
- def from_lists(cls, temperatures: List[float], top_ps: List[float],
529
- top_ks: List[int], min_ps: List[float],
530
- presence_penalties: List[float],
531
- frequency_penalties: List[float],
532
- repetition_penalties: List[float],
533
- sampling_seeds: List[int], sample_indices: List[int],
534
- prompt_tokens: List[array], output_tokens: List[array],
535
- vocab_size: int, extra_seeds_to_generate: int,
536
- device: torch.device,
537
- dtype: torch.dtype) -> "SamplingTensors":
487
+ def from_lists(
488
+ cls,
489
+ temperatures: List[float],
490
+ top_ps: List[float],
491
+ top_ks: List[int],
492
+ min_ps: List[float],
493
+ presence_penalties: List[float],
494
+ frequency_penalties: List[float],
495
+ repetition_penalties: List[float],
496
+ prompt_tokens: List[array],
497
+ output_tokens: List[array],
498
+ vocab_size: int,
499
+ device: torch.device,
500
+ dtype: torch.dtype,
501
+ ) -> "SamplingTensors":
502
# Note that the performance will be very bad without
503
# pinned memory.
504
pin_memory = is_pin_memory_available()
@@ -603,34 +567,9 @@ class SamplingTensors:
567
dtype=torch.int,
568
pin_memory=pin_memory,
569
)
606
- sample_indices_t = torch.tensor(
607
- sample_indices,
608
- device="cpu",
609
- dtype=torch.long,
610
- pin_memory=pin_memory,
611
- )
612
- # need to transpose and make contiguous to
613
- # copy the tensor correctly.
614
- # [batch_size, n_seeds] -> [n_seeds, batch_size]
615
- sampling_seeds_t = torch.tensor(
616
- sampling_seeds,
617
- device="cpu",
618
- dtype=torch.long,
619
- pin_memory=pin_memory,
620
- ).t().contiguous()
621
-
570
# Because the memory is pinned, we can do non-blocking
571
# transfer to device.
572
625
- # How many seeds the sample operation itself will need.
626
- num_base_seeds = sampling_seeds_t.shape[0] - extra_seeds_to_generate
627
- sampling_seeds_gpu = sampling_seeds_t.to(device=device,
628
- non_blocking=True)
629
- extra_seeds_gpu = sampling_seeds_gpu[num_base_seeds:]
630
- if not extra_seeds_gpu.numel():
631
- extra_seeds_gpu = None
632
- sampling_seeds_gpu = sampling_seeds_gpu[:num_base_seeds]
633
-
573
return cls(
574
temperatures=temperatures_t.to(device=device, non_blocking=True),
575
top_ps=top_ps_t.to(device=device, non_blocking=True),
@@ -644,38 +583,4 @@ class SamplingTensors:
583
non_blocking=True),
584
prompt_tokens=prompt_t.to(device=device, non_blocking=True),
585
output_tokens=output_t.to(device=device, non_blocking=True),
647
- sampling_seeds=sampling_seeds_gpu,
648
- sample_indices=sample_indices_t.to(device=device,
649
- non_blocking=True),
650
- extra_seeds=extra_seeds_gpu,
586
)
652
-
653
- @staticmethod
654
- def _get_sequence_seeds(
655
- seed: int,
656
- *extra_entropy: int,
657
- seeds_to_generate: int,
658
- is_greedy: bool,
659
- ):
660
- """Get `seeds_to_generate` child seeds from `seed` and extra entropy."""
661
- if not is_greedy:
662
- if seed is None:
663
- randint_fn = random.randint
664
- else:
665
- generator = random.Random(str((seed, ) + extra_entropy))
666
- randint_fn = generator.randint
667
- lo, hi = torch.iinfo(torch.long).min, torch.iinfo(torch.long).max
668
- # If the user/random sets seed = 0 but request should
669
- # have sampling, we need to change it to something
670
- # else. We use a constant in that case.
671
- # This way we don't need to create and load a bool
672
- # matrix in the sampling kernel, which reduces CPU
673
- # overhead and latency.
674
- seq_seeds = [
675
- randint_fn(lo, hi) or _SEED_0_REPLACEMENT
676
- for _ in range(seeds_to_generate)
677
- ]
678
- else:
679
- # For the kernel, seed == 0 means greedy decoding.
680
- seq_seeds = [0] * seeds_to_generate
681
- return seq_seeds
vllm/triton_utils/sample.py
deleted
-13
@@ -1,13 +0,0 @@
1
-import math
2
-
3
-# This is a hardcoded limit in Triton (max block size).
4
-MAX_TRITON_N_COLS = 131072
5
-
6
-
7
-def get_num_triton_sampler_splits(n_cols: int) -> int:
8
- """Get the number of splits to use for Triton sampling.
9
-
10
- Triton has a limit on the number of columns it can handle, so we need to
11
- split the tensor and call the kernel multiple times if it's too large.
12
- """
13
- return math.ceil(n_cols / MAX_TRITON_N_COLS)
vllm/utils.py
+14
-23
@@ -270,7 +270,7 @@ class LRUCache(Generic[T]):
270
271
272
class PyObjectCache:
273
- """Used to cache python objects to avoid object allocations
273
+ """Used to cache python objects to avoid object allocations
274
across scheduler iterations.
275
"""
276
@@ -289,7 +289,7 @@ class PyObjectCache:
289
self._obj_cache.append(self._obj_builder())
290
291
def get_object(self):
292
- """Returns a pre-allocated cached object. If there is not enough
292
+ """Returns a pre-allocated cached object. If there is not enough
293
objects, then the cache size will double.
294
"""
295
if self._index >= len(self._obj_cache):
@@ -837,15 +837,6 @@ def async_tensor_h2d(
837
return t.to(device=target_device, non_blocking=True)
838
839
840
-def maybe_expand_dim(tensor: torch.Tensor,
841
- target_dims: int,
842
- size: int = 1) -> torch.Tensor:
843
- """Expand the tensor to the target_dims."""
844
- if tensor.ndim < target_dims:
845
- tensor = tensor.view(-1, *([size] * (target_dims - tensor.ndim)))
846
- return tensor
847
-
848
-
840
def get_dtype_size(dtype: torch.dtype) -> int:
841
"""Get the size of the data type in bytes."""
842
return torch.tensor([], dtype=dtype).element_size()
@@ -1070,7 +1061,7 @@ def _cuda_device_count_stateless(
1061
def cuda_device_count_stateless() -> int:
1062
"""Get number of CUDA devices, caching based on the value of
1063
CUDA_VISIBLE_DEVICES at the time of call.
1073
-
1064
+
1065
This should be used instead of torch.cuda.device_count()
1066
unless CUDA_VISIBLE_DEVICES has already been set to the desired
1067
value."""
@@ -1136,10 +1127,10 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1127
def _pull_args_from_config(args: List[str]) -> List[str]:
1128
"""Method to pull arguments specified in the config file
1129
into the command-line args variable.
1139
-
1140
- The arguments in config file will be inserted between
1130
+
1131
+ The arguments in config file will be inserted between
1132
the argument list.
1142
-
1133
+
1134
example:
1135
```yaml
1136
port: 12323
@@ -1150,21 +1141,21 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1141
--config config.yaml -tp 2
1142
$: args = [
1143
"serve,chat,complete",
1153
- "facebook/opt-12B",
1154
- '--config', 'config.yaml',
1144
+ "facebook/opt-12B",
1145
+ '--config', 'config.yaml',
1146
'-tp', '2'
1147
]
1148
$: args = [
1149
"serve,chat,complete",
1159
- "facebook/opt-12B",
1160
- '--port', '12323',
1161
- '--tensor-parallel-size', '4',
1150
+ "facebook/opt-12B",
1151
+ '--port', '12323',
1152
+ '--tensor-parallel-size', '4',
1153
'-tp', '2'
1154
]
1155
```
1156
1157
Please note how the config args are inserted after the sub command.
1167
- this way the order of priorities is maintained when these are args
1158
+ this way the order of priorities is maintained when these are args
1159
parsed by super().
1160
"""
1161
assert args.count(
@@ -1190,7 +1181,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1181
1182
@staticmethod
1183
def _load_config_file(file_path: str) -> List[str]:
1193
- """Loads a yaml file and returns the key value pairs as a
1184
+ """Loads a yaml file and returns the key value pairs as a
1185
flattened list with argparse like pattern
1186
```yaml
1187
port: 12323
@@ -1201,7 +1192,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1192
'--port': '12323',
1193
'--tensor-parallel-size': '4'
1194
]
1204
-
1195
+
1196
"""
1197
1198
extension: str = file_path.split('.')[-1]