A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[MISC] Replace c10::optional with std::optional (#11730)
Signed-off-by: Lu Fang <lufang@fb.com>
Lu Fang committed
Jan 4, 2025 at 17:20 UTC
4068f4b5b5dc5e2d1114be0cbb126bc44fb4e906
24 files changed
+136
-136
csrc/attention/paged_attention_v1.cu
+2
-2
@@ -53,7 +53,7 @@ void paged_attention_v1_launcher(
53
torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
54
torch::Tensor& value_cache, int num_kv_heads, float scale,
55
torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
56
- const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
56
+ const std::optional<torch::Tensor>& alibi_slopes, float k_scale,
57
float v_scale, const int tp_rank, const int blocksparse_local_blocks,
58
const int blocksparse_vert_stride, const int blocksparse_block_size,
59
const int blocksparse_head_sliding_step) {
@@ -176,7 +176,7 @@ void paged_attention_v1(
176
torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
177
torch::Tensor& seq_lens, // [num_seqs]
178
int64_t block_size, int64_t max_seq_len,
179
- const c10::optional<torch::Tensor>& alibi_slopes,
179
+ const std::optional<torch::Tensor>& alibi_slopes,
180
const std::string& kv_cache_dtype, double k_scale, double v_scale,
181
const int64_t tp_rank, const int64_t blocksparse_local_blocks,
182
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
csrc/attention/paged_attention_v2.cu
+2
-2
@@ -54,7 +54,7 @@ void paged_attention_v2_launcher(
54
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
55
torch::Tensor& value_cache, int num_kv_heads, float scale,
56
torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
57
- const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
57
+ const std::optional<torch::Tensor>& alibi_slopes, float k_scale,
58
float v_scale, const int tp_rank, const int blocksparse_local_blocks,
59
const int blocksparse_vert_stride, const int blocksparse_block_size,
60
const int blocksparse_head_sliding_step) {
@@ -187,7 +187,7 @@ void paged_attention_v2(
187
torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
188
torch::Tensor& seq_lens, // [num_seqs]
189
int64_t block_size, int64_t max_seq_len,
190
- const c10::optional<torch::Tensor>& alibi_slopes,
190
+ const std::optional<torch::Tensor>& alibi_slopes,
191
const std::string& kv_cache_dtype, double k_scale, double v_scale,
192
const int64_t tp_rank, const int64_t blocksparse_local_blocks,
193
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
csrc/cpu/attention.cpp
+4
-4
@@ -386,7 +386,7 @@ void paged_attention_v1_impl_launcher(
386
torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
387
torch::Tensor& value_cache, int num_kv_heads, float scale,
388
torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
389
- const c10::optional<torch::Tensor>& alibi_slopes) {
389
+ const std::optional<torch::Tensor>& alibi_slopes) {
390
int num_seqs = query.size(0);
391
int num_heads = query.size(1);
392
int head_size = query.size(2);
@@ -459,7 +459,7 @@ void paged_attention_v1(
459
torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
460
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
461
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
462
- int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
462
+ int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
463
const std::string& kv_cache_dtype, double k_scale, double v_scale,
464
const int64_t tp_rank, const int64_t blocksparse_local_blocks,
465
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
@@ -702,7 +702,7 @@ void paged_attention_v2_impl_launcher(
702
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
703
torch::Tensor& value_cache, int num_kv_heads, float scale,
704
torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
705
- int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes) {
705
+ int max_seq_len, const std::optional<torch::Tensor>& alibi_slopes) {
706
int num_seqs = query.size(0);
707
int num_heads = query.size(1);
708
int head_size = query.size(2);
@@ -781,7 +781,7 @@ void paged_attention_v2(
781
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
782
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
783
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
784
- int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
784
+ int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
785
const std::string& kv_cache_dtype, double k_scale, double v_scale,
786
const int64_t tp_rank, const int64_t blocksparse_local_blocks,
787
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
csrc/cpu/quant.cpp
+5
-5
@@ -359,7 +359,7 @@ void int8_scaled_mm(torch::Tensor& c, // [M, OC], row-major
359
const torch::Tensor& b, // [IC, OC], column-major
360
const torch::Tensor& a_scales, // [1] or [M]
361
const torch::Tensor& b_scales, // [1] or [OC]
362
- const c10::optional<torch::Tensor>& bias // [OC]
362
+ const std::optional<torch::Tensor>& bias // [OC]
363
) {
364
CPU_KERNEL_GUARD_IN(cutlass_scaled_mm)
365
// Checks for conformality
@@ -442,8 +442,8 @@ void int8_scaled_mm_azp(torch::Tensor& c, // [M, OC], row-major
442
const torch::Tensor& a_scales, // [1] or [M]
443
const torch::Tensor& b_scales, // [1] or [OC]
444
const torch::Tensor& azp_adj, // [OC]
445
- const c10::optional<torch::Tensor>& azp, // [1] or [M]
446
- const c10::optional<torch::Tensor>& bias // [OC]
445
+ const std::optional<torch::Tensor>& azp, // [1] or [M]
446
+ const std::optional<torch::Tensor>& bias // [OC]
447
) {
448
CPU_KERNEL_GUARD_IN(cutlass_scaled_mm_azp)
449
// Checks for conformality
@@ -561,7 +561,7 @@ void int8_scaled_mm_azp(torch::Tensor& c, // [M, OC], row-major
561
void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
562
const torch::Tensor& input, // [..., hidden_size]
563
const torch::Tensor& scale,
564
- c10::optional<torch::Tensor> const& azp) {
564
+ std::optional<torch::Tensor> const& azp) {
565
CPU_KERNEL_GUARD_IN(static_scaled_int8_quant)
566
TORCH_CHECK(input.is_contiguous());
567
TORCH_CHECK(out.is_contiguous());
@@ -590,7 +590,7 @@ void dynamic_scaled_int8_quant(
590
torch::Tensor& out, // [..., hidden_size]
591
const torch::Tensor& input, // [..., hidden_size]
592
torch::Tensor& scale, // [..., 1]
593
- c10::optional<torch::Tensor> const& azp) {
593
+ std::optional<torch::Tensor> const& azp) {
594
CPU_KERNEL_GUARD_IN(dynamic_scaled_int8_quant)
595
TORCH_CHECK(input.is_contiguous());
596
TORCH_CHECK(out.is_contiguous());
csrc/cpu/torch_bindings.cpp
+3
-3
@@ -9,14 +9,14 @@ std::string init_cpu_threads_env(const std::string& cpu_ids);
9
void int8_scaled_mm(torch::Tensor& c, const torch::Tensor& a,
10
const torch::Tensor& b, const torch::Tensor& a_scales,
11
const torch::Tensor& b_scales,
12
- const c10::optional<torch::Tensor>& bias);
12
+ const std::optional<torch::Tensor>& bias);
13
14
void int8_scaled_mm_azp(torch::Tensor& c, const torch::Tensor& a,
15
const torch::Tensor& b, const torch::Tensor& a_scales,
16
const torch::Tensor& b_scales,
17
const torch::Tensor& azp_adj,
18
- const c10::optional<torch::Tensor>& azp,
19
- const c10::optional<torch::Tensor>& bias);
18
+ const std::optional<torch::Tensor>& azp,
19
+ const std::optional<torch::Tensor>& bias);
20
21
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
22
// vLLM custom ops
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c2x.hpp
+3
-3
@@ -68,7 +68,7 @@ struct ScaledEpilogueBase {
68
// This overload handles the case where there might not be a tensor, in which
69
// case a nullptr is passed and a constant (0) is used.
70
template <typename Descriptor, typename T>
71
- static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
71
+ static auto args_from_tensor(std::optional<torch::Tensor> const& tensor) {
72
static_assert(std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
73
using Arguments = typename Descriptor::Arguments;
74
auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
@@ -223,7 +223,7 @@ struct ScaledEpilogueBiasAzp
223
static ArgumentType prepare_args(torch::Tensor const& a_scales,
224
torch::Tensor const& b_scales,
225
torch::Tensor const& azp_adj,
226
- c10::optional<torch::Tensor> const& bias) {
226
+ std::optional<torch::Tensor> const& bias) {
227
auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
228
auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
229
auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
@@ -301,7 +301,7 @@ struct ScaledEpilogueBiasAzpToken
301
torch::Tensor const& b_scales,
302
torch::Tensor const& azp_adj,
303
torch::Tensor const& azp,
304
- c10::optional<torch::Tensor> const& bias) {
304
+ std::optional<torch::Tensor> const& bias) {
305
auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
306
auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
307
auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp
+3
-3
@@ -67,7 +67,7 @@ struct ScaledEpilogueBase {
67
// This overload handles the case where there might not be a tensor, in which
68
// case a nullptr is passed and a constant (0) is used.
69
template <typename Descriptor, typename T>
70
- static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
70
+ static auto args_from_tensor(std::optional<torch::Tensor> const& tensor) {
71
using Arguments = typename Descriptor::Arguments;
72
auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
73
static_assert(std::is_same_v<Descriptor, ColLoad<T, true>> ||
@@ -223,7 +223,7 @@ struct ScaledEpilogueBiasAzp
223
static ArgumentType prepare_args(torch::Tensor const& a_scales,
224
torch::Tensor const& b_scales,
225
torch::Tensor const& azp_adj,
226
- c10::optional<torch::Tensor> const& bias) {
226
+ std::optional<torch::Tensor> const& bias) {
227
auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
228
auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
229
auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
@@ -299,7 +299,7 @@ struct ScaledEpilogueBiasAzpToken
299
torch::Tensor const& b_scales,
300
torch::Tensor const& azp_adj,
301
torch::Tensor const& azp,
302
- c10::optional<torch::Tensor> const& bias) {
302
+ std::optional<torch::Tensor> const& bias) {
303
auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
304
auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
305
auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
csrc/cutlass_extensions/torch_utils.hpp
+1
-1
@@ -97,7 +97,7 @@ static inline auto make_cute_layout(torch::Tensor const& tensor,
97
98
template <typename Stride>
99
static inline auto maybe_make_cute_layout(
100
- c10::optional<torch::Tensor> const& tensor,
100
+ std::optional<torch::Tensor> const& tensor,
101
std::string_view name = "tensor") {
102
using Layout = decltype(make_cute_layout<Stride>(*tensor));
103
csrc/mamba/causal_conv1d/causal_conv1d.cu
+12
-12
@@ -53,12 +53,12 @@ void set_conv_params_fwd(ConvParamsBase ¶ms,
53
const at::Tensor x,
54
const at::Tensor weight,
55
const at::Tensor out,
56
- const c10::optional<at::Tensor>& bias,
56
+ const std::optional<at::Tensor>& bias,
57
bool silu_activation,
58
int64_t pad_slot_id,
59
- const c10::optional<at::Tensor>& query_start_loc = std::nullopt,
60
- const c10::optional<at::Tensor>& cache_indices = std::nullopt,
61
- const c10::optional<at::Tensor>& has_initial_state = std::nullopt) {
59
+ const std::optional<at::Tensor>& query_start_loc = std::nullopt,
60
+ const std::optional<at::Tensor>& cache_indices = std::nullopt,
61
+ const std::optional<at::Tensor>& has_initial_state = std::nullopt) {
62
63
// Reset the parameters
64
memset(¶ms, 0, sizeof(params));
@@ -93,11 +93,11 @@ void set_conv_params_fwd(ConvParamsBase ¶ms,
93
94
95
void causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
96
- const c10::optional<at::Tensor> &bias_,
97
- const c10::optional<at::Tensor> &conv_states,
98
- const c10::optional<at::Tensor> &query_start_loc,
99
- const c10::optional<at::Tensor> &cache_indices,
100
- const c10::optional<at::Tensor> &has_initial_state,
96
+ const std::optional<at::Tensor> &bias_,
97
+ const std::optional<at::Tensor> &conv_states,
98
+ const std::optional<at::Tensor> &query_start_loc,
99
+ const std::optional<at::Tensor> &cache_indices,
100
+ const std::optional<at::Tensor> &has_initial_state,
101
bool silu_activation,
102
// used to identify padding entries if cache_indices provided
103
// in case of padding, the kernel will return early
@@ -194,10 +194,10 @@ void causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
194
void causal_conv1d_update(const at::Tensor &x,
195
const at::Tensor &conv_state,
196
const at::Tensor &weight,
197
- const c10::optional<at::Tensor> &bias_,
197
+ const std::optional<at::Tensor> &bias_,
198
bool silu_activation,
199
- const c10::optional<at::Tensor> &cache_seqlens_,
200
- const c10::optional<at::Tensor> &conv_state_indices_,
199
+ const std::optional<at::Tensor> &cache_seqlens_,
200
+ const std::optional<at::Tensor> &conv_state_indices_,
201
// used to identify padding entries if cache_indices provided
202
// in case of padding, the kernel will return early
203
int64_t pad_slot_id) {
csrc/mamba/mamba_ssm/selective_scan_fwd.cu
+11
-11
@@ -402,14 +402,14 @@ void set_ssm_params_fwd(SSMParamsBase ¶ms,
402
const torch::Tensor out,
403
const torch::Tensor z,
404
const torch::Tensor out_z,
405
- const c10::optional<at::Tensor>& D,
406
- const c10::optional<at::Tensor>& delta_bias,
405
+ const std::optional<at::Tensor>& D,
406
+ const std::optional<at::Tensor>& delta_bias,
407
const torch::Tensor ssm_states,
408
bool has_z,
409
bool delta_softplus,
410
- const c10::optional<at::Tensor>& query_start_loc,
411
- const c10::optional<at::Tensor>& cache_indices,
412
- const c10::optional<at::Tensor>& has_initial_state,
410
+ const std::optional<at::Tensor>& query_start_loc,
411
+ const std::optional<at::Tensor>& cache_indices,
412
+ const std::optional<at::Tensor>& has_initial_state,
413
bool varlen,
414
int64_t pad_slot_id) {
415
@@ -504,13 +504,13 @@ void set_ssm_params_fwd(SSMParamsBase ¶ms,
504
505
void selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
506
const torch::Tensor &A, const torch::Tensor &B, const torch::Tensor &C,
507
- const c10::optional<torch::Tensor> &D_,
508
- const c10::optional<torch::Tensor> &z_,
509
- const c10::optional<torch::Tensor> &delta_bias_,
507
+ const std::optional<torch::Tensor> &D_,
508
+ const std::optional<torch::Tensor> &z_,
509
+ const std::optional<torch::Tensor> &delta_bias_,
510
bool delta_softplus,
511
- const c10::optional<torch::Tensor> &query_start_loc,
512
- const c10::optional<torch::Tensor> &cache_indices,
513
- const c10::optional<torch::Tensor> &has_initial_state,
511
+ const std::optional<torch::Tensor> &query_start_loc,
512
+ const std::optional<torch::Tensor> &cache_indices,
513
+ const std::optional<torch::Tensor> &has_initial_state,
514
const torch::Tensor &ssm_states,
515
// used to identify padding entries if cache_indices provided
516
// in case of padding, the kernel will return early
csrc/ops.h
+23
-23
@@ -33,7 +33,7 @@ void paged_attention_v1(
33
torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
34
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
35
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
36
- int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
36
+ int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
37
const std::string& kv_cache_dtype, double k_scale, double v_scale,
38
const int64_t tp_rank, const int64_t blocksparse_local_blocks,
39
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
@@ -44,7 +44,7 @@ void paged_attention_v2(
44
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
45
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
46
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
47
- int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
47
+ int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
48
const std::string& kv_cache_dtype, double k_scale, double v_scale,
49
const int64_t tp_rank, const int64_t blocksparse_local_blocks,
50
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
@@ -153,15 +153,15 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
153
void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
154
torch::Tensor const& b, torch::Tensor const& a_scales,
155
torch::Tensor const& b_scales,
156
- c10::optional<torch::Tensor> const& bias);
156
+ std::optional<torch::Tensor> const& bias);
157
158
void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
159
torch::Tensor const& b,
160
torch::Tensor const& a_scales,
161
torch::Tensor const& b_scales,
162
torch::Tensor const& azp_adj,
163
- c10::optional<torch::Tensor> const& azp,
164
- c10::optional<torch::Tensor> const& bias);
163
+ std::optional<torch::Tensor> const& azp,
164
+ std::optional<torch::Tensor> const& bias);
165
166
bool cutlass_sparse_scaled_mm_supported(int64_t cuda_device_capability);
167
@@ -169,7 +169,7 @@ void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
169
torch::Tensor const& b, torch::Tensor const& e,
170
torch::Tensor const& a_scales,
171
torch::Tensor const& b_scales,
172
- c10::optional<torch::Tensor> const& bias);
172
+ std::optional<torch::Tensor> const& bias);
173
174
bool cutlass_sparse_compress_entry(torch::Tensor& a_compressed,
175
torch::Tensor& e, torch::Tensor const& a);
@@ -177,11 +177,11 @@ bool cutlass_sparse_compress_entry(torch::Tensor& a_compressed,
177
178
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
179
torch::Tensor const& scale,
180
- c10::optional<torch::Tensor> const& azp);
180
+ std::optional<torch::Tensor> const& azp);
181
182
void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
183
torch::Tensor& scales,
184
- c10::optional<torch::Tensor> const& azp);
184
+ std::optional<torch::Tensor> const& azp);
185
186
torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
187
torch::Tensor b_gptq_qzeros,
@@ -198,34 +198,34 @@ void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
198
199
void dynamic_per_token_scaled_fp8_quant(
200
torch::Tensor& out, torch::Tensor const& input, torch::Tensor& scale,
201
- c10::optional<torch::Tensor> const& scale_ub);
201
+ std::optional<torch::Tensor> const& scale_ub);
202
203
void selective_scan_fwd(const torch::Tensor& u, const torch::Tensor& delta,
204
const torch::Tensor& A, const torch::Tensor& B,
205
const torch::Tensor& C,
206
- const c10::optional<torch::Tensor>& D_,
207
- const c10::optional<torch::Tensor>& z_,
208
- const c10::optional<torch::Tensor>& delta_bias_,
206
+ const std::optional<torch::Tensor>& D_,
207
+ const std::optional<torch::Tensor>& z_,
208
+ const std::optional<torch::Tensor>& delta_bias_,
209
bool delta_softplus,
210
- const c10::optional<torch::Tensor>& query_start_loc,
211
- const c10::optional<torch::Tensor>& cache_indices,
212
- const c10::optional<torch::Tensor>& has_initial_state,
210
+ const std::optional<torch::Tensor>& query_start_loc,
211
+ const std::optional<torch::Tensor>& cache_indices,
212
+ const std::optional<torch::Tensor>& has_initial_state,
213
const torch::Tensor& ssm_states, int64_t pad_slot_id);
214
215
void causal_conv1d_update(const at::Tensor& x, const at::Tensor& conv_state,
216
const at::Tensor& weight,
217
- const c10::optional<at::Tensor>& bias_,
217
+ const std::optional<at::Tensor>& bias_,
218
bool silu_activation,
219
- const c10::optional<at::Tensor>& cache_seqlens_,
220
- const c10::optional<at::Tensor>& conv_state_indices_,
219
+ const std::optional<at::Tensor>& cache_seqlens_,
220
+ const std::optional<at::Tensor>& conv_state_indices_,
221
int64_t pad_slot_id);
222
223
void causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
224
- const c10::optional<at::Tensor>& bias_,
225
- const c10::optional<at::Tensor>& conv_states,
226
- const c10::optional<at::Tensor>& query_start_loc,
227
- const c10::optional<at::Tensor>& cache_indices,
228
- const c10::optional<at::Tensor>& has_initial_state,
224
+ const std::optional<at::Tensor>& bias_,
225
+ const std::optional<at::Tensor>& conv_states,
226
+ const std::optional<at::Tensor>& query_start_loc,
227
+ const std::optional<at::Tensor>& cache_indices,
228
+ const std::optional<at::Tensor>& has_initial_state,
229
bool silu_activation, int64_t pad_slot_id);
230
231
#ifndef USE_ROCM
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+2
-2
@@ -226,7 +226,7 @@ __global__ void dynamic_scaled_int8_azp_quant_kernel(
226
void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
227
torch::Tensor const& input, // [..., hidden_size]
228
torch::Tensor const& scale,
229
- c10::optional<torch::Tensor> const& azp) {
229
+ std::optional<torch::Tensor> const& azp) {
230
TORCH_CHECK(input.is_contiguous());
231
TORCH_CHECK(out.is_contiguous());
232
TORCH_CHECK(scale.numel() == 1);
@@ -257,7 +257,7 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
257
void dynamic_scaled_int8_quant(
258
torch::Tensor& out, // [..., hidden_size]
259
torch::Tensor const& input, // [..., hidden_size]
260
- torch::Tensor& scales, c10::optional<torch::Tensor> const& azp) {
260
+ torch::Tensor& scales, std::optional<torch::Tensor> const& azp) {
261
TORCH_CHECK(input.is_contiguous());
262
TORCH_CHECK(out.is_contiguous());
263
TORCH_CHECK(scales.is_contiguous());
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+9
-9
@@ -39,7 +39,7 @@ void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
39
torch::Tensor const& b,
40
torch::Tensor const& a_scales,
41
torch::Tensor const& b_scales,
42
- c10::optional<torch::Tensor> const& bias) {
42
+ std::optional<torch::Tensor> const& bias) {
43
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
44
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
45
if (bias) {
@@ -58,8 +58,8 @@ void cutlass_scaled_mm_azp_sm75(torch::Tensor& out, torch::Tensor const& a,
58
torch::Tensor const& a_scales,
59
torch::Tensor const& b_scales,
60
torch::Tensor const& azp_adj,
61
- c10::optional<torch::Tensor> const& azp,
62
- c10::optional<torch::Tensor> const& bias) {
61
+ std::optional<torch::Tensor> const& azp,
62
+ std::optional<torch::Tensor> const& bias) {
63
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
64
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
65
@@ -94,7 +94,7 @@ void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
94
torch::Tensor const& b,
95
torch::Tensor const& a_scales,
96
torch::Tensor const& b_scales,
97
- c10::optional<torch::Tensor> const& bias) {
97
+ std::optional<torch::Tensor> const& bias) {
98
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
99
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
100
if (bias) {
@@ -113,8 +113,8 @@ void cutlass_scaled_mm_azp_sm80(torch::Tensor& out, torch::Tensor const& a,
113
torch::Tensor const& a_scales,
114
torch::Tensor const& b_scales,
115
torch::Tensor const& azp_adj,
116
- c10::optional<torch::Tensor> const& azp,
117
- c10::optional<torch::Tensor> const& bias) {
116
+ std::optional<torch::Tensor> const& azp,
117
+ std::optional<torch::Tensor> const& bias) {
118
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
119
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
120
@@ -165,7 +165,7 @@ void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
165
torch::Tensor const& b,
166
torch::Tensor const& a_scales,
167
torch::Tensor const& b_scales,
168
- c10::optional<torch::Tensor> const& bias) {
168
+ std::optional<torch::Tensor> const& bias) {
169
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
170
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
171
if (bias) {
@@ -184,8 +184,8 @@ void cutlass_scaled_mm_azp_sm89(torch::Tensor& out, torch::Tensor const& a,
184
torch::Tensor const& a_scales,
185
torch::Tensor const& b_scales,
186
torch::Tensor const& azp_adj,
187
- c10::optional<torch::Tensor> const& azp,
188
- c10::optional<torch::Tensor> const& bias) {
187
+ std::optional<torch::Tensor> const& azp,
188
+ std::optional<torch::Tensor> const& bias) {
189
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
190
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
191
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+3
-3
@@ -51,7 +51,7 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
51
torch::Tensor const& b,
52
torch::Tensor const& a_scales,
53
torch::Tensor const& b_scales,
54
- c10::optional<torch::Tensor> const& bias) {
54
+ std::optional<torch::Tensor> const& bias) {
55
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
56
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
57
if (bias) {
@@ -70,8 +70,8 @@ void cutlass_scaled_mm_azp_sm90(torch::Tensor& out, torch::Tensor const& a,
70
torch::Tensor const& a_scales,
71
torch::Tensor const& b_scales,
72
torch::Tensor const& azp_adj,
73
- c10::optional<torch::Tensor> const& azp,
74
- c10::optional<torch::Tensor> const& bias) {
73
+ std::optional<torch::Tensor> const& azp,
74
+ std::optional<torch::Tensor> const& bias) {
75
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
76
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
77
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+15
-15
@@ -9,26 +9,26 @@ void cutlass_scaled_mm_sm75(torch::Tensor& c, torch::Tensor const& a,
9
torch::Tensor const& b,
10
torch::Tensor const& a_scales,
11
torch::Tensor const& b_scales,
12
- c10::optional<torch::Tensor> const& bias);
12
+ std::optional<torch::Tensor> const& bias);
13
14
void cutlass_scaled_mm_sm80(torch::Tensor& c, torch::Tensor const& a,
15
torch::Tensor const& b,
16
torch::Tensor const& a_scales,
17
torch::Tensor const& b_scales,
18
- c10::optional<torch::Tensor> const& bias);
18
+ std::optional<torch::Tensor> const& bias);
19
20
void cutlass_scaled_mm_sm89(torch::Tensor& c, torch::Tensor const& a,
21
torch::Tensor const& b,
22
torch::Tensor const& a_scales,
23
torch::Tensor const& b_scales,
24
- c10::optional<torch::Tensor> const& bias);
24
+ std::optional<torch::Tensor> const& bias);
25
26
#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
27
void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
28
torch::Tensor const& b,
29
torch::Tensor const& a_scales,
30
torch::Tensor const& b_scales,
31
- c10::optional<torch::Tensor> const& bias);
31
+ std::optional<torch::Tensor> const& bias);
32
#endif
33
34
void cutlass_scaled_mm_azp_sm75(torch::Tensor& c, torch::Tensor const& a,
@@ -36,24 +36,24 @@ void cutlass_scaled_mm_azp_sm75(torch::Tensor& c, torch::Tensor const& a,
36
torch::Tensor const& a_scales,
37
torch::Tensor const& b_scales,
38
torch::Tensor const& azp_adj,
39
- c10::optional<torch::Tensor> const& azp,
40
- c10::optional<torch::Tensor> const& bias);
39
+ std::optional<torch::Tensor> const& azp,
40
+ std::optional<torch::Tensor> const& bias);
41
42
void cutlass_scaled_mm_azp_sm80(torch::Tensor& c, torch::Tensor const& a,
43
torch::Tensor const& b,
44
torch::Tensor const& a_scales,
45
torch::Tensor const& b_scales,
46
torch::Tensor const& azp_adj,
47
- c10::optional<torch::Tensor> const& azp,
48
- c10::optional<torch::Tensor> const& bias);
47
+ std::optional<torch::Tensor> const& azp,
48
+ std::optional<torch::Tensor> const& bias);
49
50
void cutlass_scaled_mm_azp_sm89(torch::Tensor& c, torch::Tensor const& a,
51
torch::Tensor const& b,
52
torch::Tensor const& a_scales,
53
torch::Tensor const& b_scales,
54
torch::Tensor const& azp_adj,
55
- c10::optional<torch::Tensor> const& azp,
56
- c10::optional<torch::Tensor> const& bias);
55
+ std::optional<torch::Tensor> const& azp,
56
+ std::optional<torch::Tensor> const& bias);
57
58
#if defined CUDA_VERSION && CUDA_VERSION >= 12000
59
void cutlass_scaled_mm_azp_sm90(torch::Tensor& c, torch::Tensor const& a,
@@ -61,8 +61,8 @@ void cutlass_scaled_mm_azp_sm90(torch::Tensor& c, torch::Tensor const& a,
61
torch::Tensor const& a_scales,
62
torch::Tensor const& b_scales,
63
torch::Tensor const& azp_adj,
64
- c10::optional<torch::Tensor> const& azp,
65
- c10::optional<torch::Tensor> const& bias);
64
+ std::optional<torch::Tensor> const& azp,
65
+ std::optional<torch::Tensor> const& bias);
66
#endif
67
68
bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
@@ -84,7 +84,7 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
84
void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
85
torch::Tensor const& b, torch::Tensor const& a_scales,
86
torch::Tensor const& b_scales,
87
- c10::optional<torch::Tensor> const& bias) {
87
+ std::optional<torch::Tensor> const& bias) {
88
// Checks for conformality
89
TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
90
TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
@@ -148,8 +148,8 @@ void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a,
148
torch::Tensor const& a_scales,
149
torch::Tensor const& b_scales,
150
torch::Tensor const& azp_adj,
151
- c10::optional<torch::Tensor> const& azp,
152
- c10::optional<torch::Tensor> const& bias) {
151
+ std::optional<torch::Tensor> const& azp,
152
+ std::optional<torch::Tensor> const& bias) {
153
// Checks for conformality
154
TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
155
TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
csrc/quantization/machete/generate.py
+1
-1
@@ -63,7 +63,7 @@ torch::Tensor mm_dispatch_{{type_sig}}(MMArgs args) {
63
64
65
static inline std::optional<at::ScalarType> maybe_scalartype(
66
- c10::optional<at::Tensor> const& t) {
66
+ std::optional<at::Tensor> const& t) {
67
if (!t) {
68
return std::nullopt;
69
} else {
csrc/quantization/machete/machete_mm_kernel.cuh
+5
-5
@@ -183,11 +183,11 @@ struct MacheteKernelTemplate {
183
torch::Tensor const& A, // MxK matrix
184
torch::Tensor const& B, // KxN prepacked matrix
185
torch::Tensor& D, // MxN matrix
186
- c10::optional<torch::Tensor> const& maybe_g_scales, // scale_KxN matrix
187
- c10::optional<torch::Tensor> const& maybe_g_zeros, // scale_KxN matrix
188
- c10::optional<int64_t> maybe_group_size,
189
- c10::optional<torch::Tensor> const& maybe_ch_scales, // len N vector
190
- c10::optional<torch::Tensor> const& maybe_tok_scales) // len M vector
186
+ std::optional<torch::Tensor> const& maybe_g_scales, // scale_KxN matrix
187
+ std::optional<torch::Tensor> const& maybe_g_zeros, // scale_KxN matrix
188
+ std::optional<int64_t> maybe_group_size,
189
+ std::optional<torch::Tensor> const& maybe_ch_scales, // len N vector
190
+ std::optional<torch::Tensor> const& maybe_tok_scales) // len M vector
191
{
192
static_assert(!with_group_zeropoints || with_group_scales);
193
csrc/quantization/machete/machete_mm_launcher.cuh
+12
-12
@@ -13,23 +13,23 @@ struct MMArgs {
13
torch::Tensor const& A;
14
torch::Tensor const& B;
15
vllm::ScalarType const& b_type;
16
- c10::optional<at::ScalarType> const& maybe_out_type;
17
- c10::optional<torch::Tensor> const& maybe_group_scales;
18
- c10::optional<torch::Tensor> const& maybe_group_zeros;
19
- c10::optional<int64_t> maybe_group_size;
20
- c10::optional<torch::Tensor> const& maybe_channel_scales;
21
- c10::optional<torch::Tensor> const& maybe_token_scales;
22
- c10::optional<std::string> maybe_schedule;
16
+ std::optional<at::ScalarType> const& maybe_out_type;
17
+ std::optional<torch::Tensor> const& maybe_group_scales;
18
+ std::optional<torch::Tensor> const& maybe_group_zeros;
19
+ std::optional<int64_t> maybe_group_size;
20
+ std::optional<torch::Tensor> const& maybe_channel_scales;
21
+ std::optional<torch::Tensor> const& maybe_token_scales;
22
+ std::optional<std::string> maybe_schedule;
23
};
24
25
struct SupportedSchedulesArgs {
26
at::ScalarType a_type;
27
vllm::ScalarType b_type;
28
- c10::optional<at::ScalarType> maybe_group_scales_type;
29
- c10::optional<at::ScalarType> maybe_group_zeros_type;
30
- c10::optional<at::ScalarType> maybe_channel_scales_type;
31
- c10::optional<at::ScalarType> maybe_token_scales_type;
32
- c10::optional<at::ScalarType> maybe_out_type;
28
+ std::optional<at::ScalarType> maybe_group_scales_type;
29
+ std::optional<at::ScalarType> maybe_group_zeros_type;
30
+ std::optional<at::ScalarType> maybe_channel_scales_type;
31
+ std::optional<at::ScalarType> maybe_token_scales_type;
32
+ std::optional<at::ScalarType> maybe_out_type;
33
};
34
35
torch::Tensor mm_dispatch(MMArgs args);
csrc/quantization/machete/machete_prepack_launcher.cuh
+1
-1
@@ -10,7 +10,7 @@ struct PrepackBArgs {
10
torch::Tensor const& B;
11
at::ScalarType a_type;
12
vllm::ScalarType b_type;
13
- c10::optional<at::ScalarType> maybe_group_scales_type;
13
+ std::optional<at::ScalarType> maybe_group_scales_type;
14
};
15
16
template <typename PrepackedLayoutB>
csrc/quantization/machete/machete_pytorch.cu
+13
-13
@@ -10,11 +10,11 @@ using namespace vllm;
10
11
std::vector<std::string> supported_schedules(
12
at::ScalarType a_type, int64_t b_type_id,
13
- c10::optional<at::ScalarType> maybe_group_scales_type,
14
- c10::optional<at::ScalarType> maybe_group_zeros_type,
15
- c10::optional<at::ScalarType> maybe_channel_scales_type,
16
- c10::optional<at::ScalarType> maybe_token_scales_type,
17
- c10::optional<at::ScalarType> maybe_out_type) {
13
+ std::optional<at::ScalarType> maybe_group_scales_type,
14
+ std::optional<at::ScalarType> maybe_group_zeros_type,
15
+ std::optional<at::ScalarType> maybe_channel_scales_type,
16
+ std::optional<at::ScalarType> maybe_token_scales_type,
17
+ std::optional<at::ScalarType> maybe_out_type) {
18
ScalarType const b_type = ScalarType::from_id(b_type_id);
19
return supported_schedules_dispatch({
20
.a_type = a_type,
@@ -29,13 +29,13 @@ std::vector<std::string> supported_schedules(
29
30
torch::Tensor mm(torch::Tensor const& A, torch::Tensor const& B,
31
int64_t b_type_id,
32
- c10::optional<at::ScalarType> const& maybe_out_type,
33
- c10::optional<torch::Tensor> const& maybe_group_scales,
34
- c10::optional<torch::Tensor> const& maybe_group_zeros,
35
- c10::optional<int64_t> maybe_group_size,
36
- c10::optional<torch::Tensor> const& maybe_channel_scales,
37
- c10::optional<torch::Tensor> const& maybe_token_scales,
38
- c10::optional<std::string> maybe_schedule) {
32
+ std::optional<at::ScalarType> const& maybe_out_type,
33
+ std::optional<torch::Tensor> const& maybe_group_scales,
34
+ std::optional<torch::Tensor> const& maybe_group_zeros,
35
+ std::optional<int64_t> maybe_group_size,
36
+ std::optional<torch::Tensor> const& maybe_channel_scales,
37
+ std::optional<torch::Tensor> const& maybe_token_scales,
38
+ std::optional<std::string> maybe_schedule) {
39
ScalarType const b_type = ScalarType::from_id(b_type_id);
40
return mm_dispatch({.A = A,
41
.B = B,
@@ -51,7 +51,7 @@ torch::Tensor mm(torch::Tensor const& A, torch::Tensor const& B,
51
52
torch::Tensor prepack_B(
53
torch::Tensor const& B, at::ScalarType const& a_type, int64_t b_type_id,
54
- c10::optional<at::ScalarType> const& maybe_group_scales_type) {
54
+ std::optional<at::ScalarType> const& maybe_group_scales_type) {
55
ScalarType const b_type = ScalarType::from_id(b_type_id);
56
return prepack_B_dispatch(
57
{.B = B,
csrc/rocm/attention.cu
+2
-2
@@ -928,7 +928,7 @@ void paged_attention_custom_launcher(
928
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
929
torch::Tensor& value_cache, const int num_kv_heads, float scale,
930
torch::Tensor& block_tables, torch::Tensor& context_lens,
931
- int max_context_len, const c10::optional<torch::Tensor>& alibi_slopes,
931
+ int max_context_len, const std::optional<torch::Tensor>& alibi_slopes,
932
float k_scale, float v_scale) {
933
int num_seqs = query.size(0);
934
int num_heads = query.size(1);
@@ -1086,7 +1086,7 @@ void paged_attention(
1086
torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
1087
torch::Tensor& context_lens, // [num_seqs]
1088
int64_t block_size, int64_t max_context_len,
1089
- const c10::optional<torch::Tensor>& alibi_slopes,
1089
+ const std::optional<torch::Tensor>& alibi_slopes,
1090
const std::string& kv_cache_dtype, double k_scale, double v_scale) {
1091
const int head_size = query.size(2);
1092
if (kv_cache_dtype == "auto") {
csrc/rocm/ops.h
+1
-1
@@ -9,6 +9,6 @@ void paged_attention(torch::Tensor& out, torch::Tensor& exp_sums,
9
double scale, torch::Tensor& block_tables,
10
torch::Tensor& context_lens, int64_t block_size,
11
int64_t max_context_len,
12
- const c10::optional<torch::Tensor>& alibi_slopes,
12
+ const std::optional<torch::Tensor>& alibi_slopes,
13
const std::string& kv_cache_dtype, double k_scale,
14
double v_scale);
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu
+1
-1
@@ -286,7 +286,7 @@ void cutlass_scaled_sparse_mm_sm90(torch::Tensor& out, torch::Tensor const& a,
286
torch::Tensor const& bt_meta,
287
torch::Tensor const& a_scales,
288
torch::Tensor const& b_scales,
289
- c10::optional<torch::Tensor> const& bias) {
289
+ std::optional<torch::Tensor> const& bias) {
290
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
291
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
292
if (bias) {
csrc/sparse/cutlass/sparse_scaled_mm_entry.cu
+2
-2
@@ -22,7 +22,7 @@ void cutlass_scaled_sparse_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
22
torch::Tensor const& e,
23
torch::Tensor const& a_scales,
24
torch::Tensor const& b_scales,
25
- c10::optional<torch::Tensor> const& bias);
25
+ std::optional<torch::Tensor> const& bias);
26
#endif
27
28
void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
@@ -30,7 +30,7 @@ void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
30
torch::Tensor const& bt_meta,
31
torch::Tensor const& a_scales,
32
torch::Tensor const& b_scales,
33
- c10::optional<torch::Tensor> const& bias) {
33
+ std::optional<torch::Tensor> const& bias) {
34
// Checks for conformality
35
TORCH_CHECK(a.dim() == 2 && bt_nzs.dim() == 2 && c.dim() == 2);
36
TORCH_CHECK(c.size(1) == bt_nzs.size(0) && bt_nzs.size(1) * 2 == a.size(1) &&