@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4068f4b5b5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[MISC] Replace c10::optional with std::optional (#11730)

Signed-off-by: Lu Fang <lufang@fb.com>

Lu Fang committed Jan 4, 2025 at 17:20 UTC 4068f4b5b5dc5e2d1114be0cbb126bc44fb4e906
24 files changed +136 -136
csrc/attention/paged_attention_v1.cu
+2 -2
@@ -53,7 +53,7 @@ void paged_attention_v1_launcher(
53 torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
54 torch::Tensor& value_cache, int num_kv_heads, float scale,
55 torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
56 - const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
56 + const std::optional<torch::Tensor>& alibi_slopes, float k_scale,
57 float v_scale, const int tp_rank, const int blocksparse_local_blocks,
58 const int blocksparse_vert_stride, const int blocksparse_block_size,
59 const int blocksparse_head_sliding_step) {
@@ -176,7 +176,7 @@ void paged_attention_v1(
176 torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
177 torch::Tensor& seq_lens, // [num_seqs]
178 int64_t block_size, int64_t max_seq_len,
179 - const c10::optional<torch::Tensor>& alibi_slopes,
179 + const std::optional<torch::Tensor>& alibi_slopes,
180 const std::string& kv_cache_dtype, double k_scale, double v_scale,
181 const int64_t tp_rank, const int64_t blocksparse_local_blocks,
182 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
csrc/attention/paged_attention_v2.cu
+2 -2
@@ -54,7 +54,7 @@ void paged_attention_v2_launcher(
54 torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
55 torch::Tensor& value_cache, int num_kv_heads, float scale,
56 torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
57 - const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
57 + const std::optional<torch::Tensor>& alibi_slopes, float k_scale,
58 float v_scale, const int tp_rank, const int blocksparse_local_blocks,
59 const int blocksparse_vert_stride, const int blocksparse_block_size,
60 const int blocksparse_head_sliding_step) {
@@ -187,7 +187,7 @@ void paged_attention_v2(
187 torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
188 torch::Tensor& seq_lens, // [num_seqs]
189 int64_t block_size, int64_t max_seq_len,
190 - const c10::optional<torch::Tensor>& alibi_slopes,
190 + const std::optional<torch::Tensor>& alibi_slopes,
191 const std::string& kv_cache_dtype, double k_scale, double v_scale,
192 const int64_t tp_rank, const int64_t blocksparse_local_blocks,
193 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
csrc/cpu/attention.cpp
+4 -4
@@ -386,7 +386,7 @@ void paged_attention_v1_impl_launcher(
386 torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
387 torch::Tensor& value_cache, int num_kv_heads, float scale,
388 torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
389 - const c10::optional<torch::Tensor>& alibi_slopes) {
389 + const std::optional<torch::Tensor>& alibi_slopes) {
390 int num_seqs = query.size(0);
391 int num_heads = query.size(1);
392 int head_size = query.size(2);
@@ -459,7 +459,7 @@ void paged_attention_v1(
459 torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
460 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
461 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
462 - int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
462 + int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
463 const std::string& kv_cache_dtype, double k_scale, double v_scale,
464 const int64_t tp_rank, const int64_t blocksparse_local_blocks,
465 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
@@ -702,7 +702,7 @@ void paged_attention_v2_impl_launcher(
702 torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
703 torch::Tensor& value_cache, int num_kv_heads, float scale,
704 torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
705 - int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes) {
705 + int max_seq_len, const std::optional<torch::Tensor>& alibi_slopes) {
706 int num_seqs = query.size(0);
707 int num_heads = query.size(1);
708 int head_size = query.size(2);
@@ -781,7 +781,7 @@ void paged_attention_v2(
781 torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
782 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
783 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
784 - int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
784 + int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
785 const std::string& kv_cache_dtype, double k_scale, double v_scale,
786 const int64_t tp_rank, const int64_t blocksparse_local_blocks,
787 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
csrc/cpu/quant.cpp
+5 -5
@@ -359,7 +359,7 @@ void int8_scaled_mm(torch::Tensor& c, // [M, OC], row-major
359 const torch::Tensor& b, // [IC, OC], column-major
360 const torch::Tensor& a_scales, // [1] or [M]
361 const torch::Tensor& b_scales, // [1] or [OC]
362 - const c10::optional<torch::Tensor>& bias // [OC]
362 + const std::optional<torch::Tensor>& bias // [OC]
363 ) {
364 CPU_KERNEL_GUARD_IN(cutlass_scaled_mm)
365 // Checks for conformality
@@ -442,8 +442,8 @@ void int8_scaled_mm_azp(torch::Tensor& c, // [M, OC], row-major
442 const torch::Tensor& a_scales, // [1] or [M]
443 const torch::Tensor& b_scales, // [1] or [OC]
444 const torch::Tensor& azp_adj, // [OC]
445 - const c10::optional<torch::Tensor>& azp, // [1] or [M]
446 - const c10::optional<torch::Tensor>& bias // [OC]
445 + const std::optional<torch::Tensor>& azp, // [1] or [M]
446 + const std::optional<torch::Tensor>& bias // [OC]
447 ) {
448 CPU_KERNEL_GUARD_IN(cutlass_scaled_mm_azp)
449 // Checks for conformality
@@ -561,7 +561,7 @@ void int8_scaled_mm_azp(torch::Tensor& c, // [M, OC], row-major
561 void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
562 const torch::Tensor& input, // [..., hidden_size]
563 const torch::Tensor& scale,
564 - c10::optional<torch::Tensor> const& azp) {
564 + std::optional<torch::Tensor> const& azp) {
565 CPU_KERNEL_GUARD_IN(static_scaled_int8_quant)
566 TORCH_CHECK(input.is_contiguous());
567 TORCH_CHECK(out.is_contiguous());
@@ -590,7 +590,7 @@ void dynamic_scaled_int8_quant(
590 torch::Tensor& out, // [..., hidden_size]
591 const torch::Tensor& input, // [..., hidden_size]
592 torch::Tensor& scale, // [..., 1]
593 - c10::optional<torch::Tensor> const& azp) {
593 + std::optional<torch::Tensor> const& azp) {
594 CPU_KERNEL_GUARD_IN(dynamic_scaled_int8_quant)
595 TORCH_CHECK(input.is_contiguous());
596 TORCH_CHECK(out.is_contiguous());
csrc/cpu/torch_bindings.cpp
+3 -3
@@ -9,14 +9,14 @@ std::string init_cpu_threads_env(const std::string& cpu_ids);
9 void int8_scaled_mm(torch::Tensor& c, const torch::Tensor& a,
10 const torch::Tensor& b, const torch::Tensor& a_scales,
11 const torch::Tensor& b_scales,
12 - const c10::optional<torch::Tensor>& bias);
12 + const std::optional<torch::Tensor>& bias);
13
14 void int8_scaled_mm_azp(torch::Tensor& c, const torch::Tensor& a,
15 const torch::Tensor& b, const torch::Tensor& a_scales,
16 const torch::Tensor& b_scales,
17 const torch::Tensor& azp_adj,
18 - const c10::optional<torch::Tensor>& azp,
19 - const c10::optional<torch::Tensor>& bias);
18 + const std::optional<torch::Tensor>& azp,
19 + const std::optional<torch::Tensor>& bias);
20
21 TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
22 // vLLM custom ops
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c2x.hpp
+3 -3
@@ -68,7 +68,7 @@ struct ScaledEpilogueBase {
68 // This overload handles the case where there might not be a tensor, in which
69 // case a nullptr is passed and a constant (0) is used.
70 template <typename Descriptor, typename T>
71 - static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
71 + static auto args_from_tensor(std::optional<torch::Tensor> const& tensor) {
72 static_assert(std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
73 using Arguments = typename Descriptor::Arguments;
74 auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
@@ -223,7 +223,7 @@ struct ScaledEpilogueBiasAzp
223 static ArgumentType prepare_args(torch::Tensor const& a_scales,
224 torch::Tensor const& b_scales,
225 torch::Tensor const& azp_adj,
226 - c10::optional<torch::Tensor> const& bias) {
226 + std::optional<torch::Tensor> const& bias) {
227 auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
228 auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
229 auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
@@ -301,7 +301,7 @@ struct ScaledEpilogueBiasAzpToken
301 torch::Tensor const& b_scales,
302 torch::Tensor const& azp_adj,
303 torch::Tensor const& azp,
304 - c10::optional<torch::Tensor> const& bias) {
304 + std::optional<torch::Tensor> const& bias) {
305 auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
306 auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
307 auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp
+3 -3
@@ -67,7 +67,7 @@ struct ScaledEpilogueBase {
67 // This overload handles the case where there might not be a tensor, in which
68 // case a nullptr is passed and a constant (0) is used.
69 template <typename Descriptor, typename T>
70 - static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
70 + static auto args_from_tensor(std::optional<torch::Tensor> const& tensor) {
71 using Arguments = typename Descriptor::Arguments;
72 auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
73 static_assert(std::is_same_v<Descriptor, ColLoad<T, true>> ||
@@ -223,7 +223,7 @@ struct ScaledEpilogueBiasAzp
223 static ArgumentType prepare_args(torch::Tensor const& a_scales,
224 torch::Tensor const& b_scales,
225 torch::Tensor const& azp_adj,
226 - c10::optional<torch::Tensor> const& bias) {
226 + std::optional<torch::Tensor> const& bias) {
227 auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
228 auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
229 auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
@@ -299,7 +299,7 @@ struct ScaledEpilogueBiasAzpToken
299 torch::Tensor const& b_scales,
300 torch::Tensor const& azp_adj,
301 torch::Tensor const& azp,
302 - c10::optional<torch::Tensor> const& bias) {
302 + std::optional<torch::Tensor> const& bias) {
303 auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
304 auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
305 auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
csrc/cutlass_extensions/torch_utils.hpp
+1 -1
@@ -97,7 +97,7 @@ static inline auto make_cute_layout(torch::Tensor const& tensor,
97
98 template <typename Stride>
99 static inline auto maybe_make_cute_layout(
100 - c10::optional<torch::Tensor> const& tensor,
100 + std::optional<torch::Tensor> const& tensor,
101 std::string_view name = "tensor") {
102 using Layout = decltype(make_cute_layout<Stride>(*tensor));
103
csrc/mamba/causal_conv1d/causal_conv1d.cu
+12 -12
@@ -53,12 +53,12 @@ void set_conv_params_fwd(ConvParamsBase &params,
53 const at::Tensor x,
54 const at::Tensor weight,
55 const at::Tensor out,
56 - const c10::optional<at::Tensor>& bias,
56 + const std::optional<at::Tensor>& bias,
57 bool silu_activation,
58 int64_t pad_slot_id,
59 - const c10::optional<at::Tensor>& query_start_loc = std::nullopt,
60 - const c10::optional<at::Tensor>& cache_indices = std::nullopt,
61 - const c10::optional<at::Tensor>& has_initial_state = std::nullopt) {
59 + const std::optional<at::Tensor>& query_start_loc = std::nullopt,
60 + const std::optional<at::Tensor>& cache_indices = std::nullopt,
61 + const std::optional<at::Tensor>& has_initial_state = std::nullopt) {
62
63 // Reset the parameters
64 memset(&params, 0, sizeof(params));
@@ -93,11 +93,11 @@ void set_conv_params_fwd(ConvParamsBase &params,
93
94
95 void causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
96 - const c10::optional<at::Tensor> &bias_,
97 - const c10::optional<at::Tensor> &conv_states,
98 - const c10::optional<at::Tensor> &query_start_loc,
99 - const c10::optional<at::Tensor> &cache_indices,
100 - const c10::optional<at::Tensor> &has_initial_state,
96 + const std::optional<at::Tensor> &bias_,
97 + const std::optional<at::Tensor> &conv_states,
98 + const std::optional<at::Tensor> &query_start_loc,
99 + const std::optional<at::Tensor> &cache_indices,
100 + const std::optional<at::Tensor> &has_initial_state,
101 bool silu_activation,
102 // used to identify padding entries if cache_indices provided
103 // in case of padding, the kernel will return early
@@ -194,10 +194,10 @@ void causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
194 void causal_conv1d_update(const at::Tensor &x,
195 const at::Tensor &conv_state,
196 const at::Tensor &weight,
197 - const c10::optional<at::Tensor> &bias_,
197 + const std::optional<at::Tensor> &bias_,
198 bool silu_activation,
199 - const c10::optional<at::Tensor> &cache_seqlens_,
200 - const c10::optional<at::Tensor> &conv_state_indices_,
199 + const std::optional<at::Tensor> &cache_seqlens_,
200 + const std::optional<at::Tensor> &conv_state_indices_,
201 // used to identify padding entries if cache_indices provided
202 // in case of padding, the kernel will return early
203 int64_t pad_slot_id) {
csrc/mamba/mamba_ssm/selective_scan_fwd.cu
+11 -11
@@ -402,14 +402,14 @@ void set_ssm_params_fwd(SSMParamsBase &params,
402 const torch::Tensor out,
403 const torch::Tensor z,
404 const torch::Tensor out_z,
405 - const c10::optional<at::Tensor>& D,
406 - const c10::optional<at::Tensor>& delta_bias,
405 + const std::optional<at::Tensor>& D,
406 + const std::optional<at::Tensor>& delta_bias,
407 const torch::Tensor ssm_states,
408 bool has_z,
409 bool delta_softplus,
410 - const c10::optional<at::Tensor>& query_start_loc,
411 - const c10::optional<at::Tensor>& cache_indices,
412 - const c10::optional<at::Tensor>& has_initial_state,
410 + const std::optional<at::Tensor>& query_start_loc,
411 + const std::optional<at::Tensor>& cache_indices,
412 + const std::optional<at::Tensor>& has_initial_state,
413 bool varlen,
414 int64_t pad_slot_id) {
415
@@ -504,13 +504,13 @@ void set_ssm_params_fwd(SSMParamsBase &params,
504
505 void selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
506 const torch::Tensor &A, const torch::Tensor &B, const torch::Tensor &C,
507 - const c10::optional<torch::Tensor> &D_,
508 - const c10::optional<torch::Tensor> &z_,
509 - const c10::optional<torch::Tensor> &delta_bias_,
507 + const std::optional<torch::Tensor> &D_,
508 + const std::optional<torch::Tensor> &z_,
509 + const std::optional<torch::Tensor> &delta_bias_,
510 bool delta_softplus,
511 - const c10::optional<torch::Tensor> &query_start_loc,
512 - const c10::optional<torch::Tensor> &cache_indices,
513 - const c10::optional<torch::Tensor> &has_initial_state,
511 + const std::optional<torch::Tensor> &query_start_loc,
512 + const std::optional<torch::Tensor> &cache_indices,
513 + const std::optional<torch::Tensor> &has_initial_state,
514 const torch::Tensor &ssm_states,
515 // used to identify padding entries if cache_indices provided
516 // in case of padding, the kernel will return early
csrc/ops.h
+23 -23
@@ -33,7 +33,7 @@ void paged_attention_v1(
33 torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
34 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
35 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
36 - int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
36 + int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
37 const std::string& kv_cache_dtype, double k_scale, double v_scale,
38 const int64_t tp_rank, const int64_t blocksparse_local_blocks,
39 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
@@ -44,7 +44,7 @@ void paged_attention_v2(
44 torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
45 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
46 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
47 - int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
47 + int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
48 const std::string& kv_cache_dtype, double k_scale, double v_scale,
49 const int64_t tp_rank, const int64_t blocksparse_local_blocks,
50 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
@@ -153,15 +153,15 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
153 void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
154 torch::Tensor const& b, torch::Tensor const& a_scales,
155 torch::Tensor const& b_scales,
156 - c10::optional<torch::Tensor> const& bias);
156 + std::optional<torch::Tensor> const& bias);
157
158 void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
159 torch::Tensor const& b,
160 torch::Tensor const& a_scales,
161 torch::Tensor const& b_scales,
162 torch::Tensor const& azp_adj,
163 - c10::optional<torch::Tensor> const& azp,
164 - c10::optional<torch::Tensor> const& bias);
163 + std::optional<torch::Tensor> const& azp,
164 + std::optional<torch::Tensor> const& bias);
165
166 bool cutlass_sparse_scaled_mm_supported(int64_t cuda_device_capability);
167
@@ -169,7 +169,7 @@ void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
169 torch::Tensor const& b, torch::Tensor const& e,
170 torch::Tensor const& a_scales,
171 torch::Tensor const& b_scales,
172 - c10::optional<torch::Tensor> const& bias);
172 + std::optional<torch::Tensor> const& bias);
173
174 bool cutlass_sparse_compress_entry(torch::Tensor& a_compressed,
175 torch::Tensor& e, torch::Tensor const& a);
@@ -177,11 +177,11 @@ bool cutlass_sparse_compress_entry(torch::Tensor& a_compressed,
177
178 void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
179 torch::Tensor const& scale,
180 - c10::optional<torch::Tensor> const& azp);
180 + std::optional<torch::Tensor> const& azp);
181
182 void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
183 torch::Tensor& scales,
184 - c10::optional<torch::Tensor> const& azp);
184 + std::optional<torch::Tensor> const& azp);
185
186 torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
187 torch::Tensor b_gptq_qzeros,
@@ -198,34 +198,34 @@ void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
198
199 void dynamic_per_token_scaled_fp8_quant(
200 torch::Tensor& out, torch::Tensor const& input, torch::Tensor& scale,
201 - c10::optional<torch::Tensor> const& scale_ub);
201 + std::optional<torch::Tensor> const& scale_ub);
202
203 void selective_scan_fwd(const torch::Tensor& u, const torch::Tensor& delta,
204 const torch::Tensor& A, const torch::Tensor& B,
205 const torch::Tensor& C,
206 - const c10::optional<torch::Tensor>& D_,
207 - const c10::optional<torch::Tensor>& z_,
208 - const c10::optional<torch::Tensor>& delta_bias_,
206 + const std::optional<torch::Tensor>& D_,
207 + const std::optional<torch::Tensor>& z_,
208 + const std::optional<torch::Tensor>& delta_bias_,
209 bool delta_softplus,
210 - const c10::optional<torch::Tensor>& query_start_loc,
211 - const c10::optional<torch::Tensor>& cache_indices,
212 - const c10::optional<torch::Tensor>& has_initial_state,
210 + const std::optional<torch::Tensor>& query_start_loc,
211 + const std::optional<torch::Tensor>& cache_indices,
212 + const std::optional<torch::Tensor>& has_initial_state,
213 const torch::Tensor& ssm_states, int64_t pad_slot_id);
214
215 void causal_conv1d_update(const at::Tensor& x, const at::Tensor& conv_state,
216 const at::Tensor& weight,
217 - const c10::optional<at::Tensor>& bias_,
217 + const std::optional<at::Tensor>& bias_,
218 bool silu_activation,
219 - const c10::optional<at::Tensor>& cache_seqlens_,
220 - const c10::optional<at::Tensor>& conv_state_indices_,
219 + const std::optional<at::Tensor>& cache_seqlens_,
220 + const std::optional<at::Tensor>& conv_state_indices_,
221 int64_t pad_slot_id);
222
223 void causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
224 - const c10::optional<at::Tensor>& bias_,
225 - const c10::optional<at::Tensor>& conv_states,
226 - const c10::optional<at::Tensor>& query_start_loc,
227 - const c10::optional<at::Tensor>& cache_indices,
228 - const c10::optional<at::Tensor>& has_initial_state,
224 + const std::optional<at::Tensor>& bias_,
225 + const std::optional<at::Tensor>& conv_states,
226 + const std::optional<at::Tensor>& query_start_loc,
227 + const std::optional<at::Tensor>& cache_indices,
228 + const std::optional<at::Tensor>& has_initial_state,
229 bool silu_activation, int64_t pad_slot_id);
230
231 #ifndef USE_ROCM
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+2 -2
@@ -226,7 +226,7 @@ __global__ void dynamic_scaled_int8_azp_quant_kernel(
226 void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
227 torch::Tensor const& input, // [..., hidden_size]
228 torch::Tensor const& scale,
229 - c10::optional<torch::Tensor> const& azp) {
229 + std::optional<torch::Tensor> const& azp) {
230 TORCH_CHECK(input.is_contiguous());
231 TORCH_CHECK(out.is_contiguous());
232 TORCH_CHECK(scale.numel() == 1);
@@ -257,7 +257,7 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
257 void dynamic_scaled_int8_quant(
258 torch::Tensor& out, // [..., hidden_size]
259 torch::Tensor const& input, // [..., hidden_size]
260 - torch::Tensor& scales, c10::optional<torch::Tensor> const& azp) {
260 + torch::Tensor& scales, std::optional<torch::Tensor> const& azp) {
261 TORCH_CHECK(input.is_contiguous());
262 TORCH_CHECK(out.is_contiguous());
263 TORCH_CHECK(scales.is_contiguous());
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+9 -9
@@ -39,7 +39,7 @@ void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
39 torch::Tensor const& b,
40 torch::Tensor const& a_scales,
41 torch::Tensor const& b_scales,
42 - c10::optional<torch::Tensor> const& bias) {
42 + std::optional<torch::Tensor> const& bias) {
43 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
44 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
45 if (bias) {
@@ -58,8 +58,8 @@ void cutlass_scaled_mm_azp_sm75(torch::Tensor& out, torch::Tensor const& a,
58 torch::Tensor const& a_scales,
59 torch::Tensor const& b_scales,
60 torch::Tensor const& azp_adj,
61 - c10::optional<torch::Tensor> const& azp,
62 - c10::optional<torch::Tensor> const& bias) {
61 + std::optional<torch::Tensor> const& azp,
62 + std::optional<torch::Tensor> const& bias) {
63 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
64 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
65
@@ -94,7 +94,7 @@ void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
94 torch::Tensor const& b,
95 torch::Tensor const& a_scales,
96 torch::Tensor const& b_scales,
97 - c10::optional<torch::Tensor> const& bias) {
97 + std::optional<torch::Tensor> const& bias) {
98 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
99 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
100 if (bias) {
@@ -113,8 +113,8 @@ void cutlass_scaled_mm_azp_sm80(torch::Tensor& out, torch::Tensor const& a,
113 torch::Tensor const& a_scales,
114 torch::Tensor const& b_scales,
115 torch::Tensor const& azp_adj,
116 - c10::optional<torch::Tensor> const& azp,
117 - c10::optional<torch::Tensor> const& bias) {
116 + std::optional<torch::Tensor> const& azp,
117 + std::optional<torch::Tensor> const& bias) {
118 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
119 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
120
@@ -165,7 +165,7 @@ void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
165 torch::Tensor const& b,
166 torch::Tensor const& a_scales,
167 torch::Tensor const& b_scales,
168 - c10::optional<torch::Tensor> const& bias) {
168 + std::optional<torch::Tensor> const& bias) {
169 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
170 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
171 if (bias) {
@@ -184,8 +184,8 @@ void cutlass_scaled_mm_azp_sm89(torch::Tensor& out, torch::Tensor const& a,
184 torch::Tensor const& a_scales,
185 torch::Tensor const& b_scales,
186 torch::Tensor const& azp_adj,
187 - c10::optional<torch::Tensor> const& azp,
188 - c10::optional<torch::Tensor> const& bias) {
187 + std::optional<torch::Tensor> const& azp,
188 + std::optional<torch::Tensor> const& bias) {
189 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
190 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
191
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+3 -3
@@ -51,7 +51,7 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
51 torch::Tensor const& b,
52 torch::Tensor const& a_scales,
53 torch::Tensor const& b_scales,
54 - c10::optional<torch::Tensor> const& bias) {
54 + std::optional<torch::Tensor> const& bias) {
55 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
56 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
57 if (bias) {
@@ -70,8 +70,8 @@ void cutlass_scaled_mm_azp_sm90(torch::Tensor& out, torch::Tensor const& a,
70 torch::Tensor const& a_scales,
71 torch::Tensor const& b_scales,
72 torch::Tensor const& azp_adj,
73 - c10::optional<torch::Tensor> const& azp,
74 - c10::optional<torch::Tensor> const& bias) {
73 + std::optional<torch::Tensor> const& azp,
74 + std::optional<torch::Tensor> const& bias) {
75 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
76 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
77
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+15 -15
@@ -9,26 +9,26 @@ void cutlass_scaled_mm_sm75(torch::Tensor& c, torch::Tensor const& a,
9 torch::Tensor const& b,
10 torch::Tensor const& a_scales,
11 torch::Tensor const& b_scales,
12 - c10::optional<torch::Tensor> const& bias);
12 + std::optional<torch::Tensor> const& bias);
13
14 void cutlass_scaled_mm_sm80(torch::Tensor& c, torch::Tensor const& a,
15 torch::Tensor const& b,
16 torch::Tensor const& a_scales,
17 torch::Tensor const& b_scales,
18 - c10::optional<torch::Tensor> const& bias);
18 + std::optional<torch::Tensor> const& bias);
19
20 void cutlass_scaled_mm_sm89(torch::Tensor& c, torch::Tensor const& a,
21 torch::Tensor const& b,
22 torch::Tensor const& a_scales,
23 torch::Tensor const& b_scales,
24 - c10::optional<torch::Tensor> const& bias);
24 + std::optional<torch::Tensor> const& bias);
25
26 #if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
27 void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
28 torch::Tensor const& b,
29 torch::Tensor const& a_scales,
30 torch::Tensor const& b_scales,
31 - c10::optional<torch::Tensor> const& bias);
31 + std::optional<torch::Tensor> const& bias);
32 #endif
33
34 void cutlass_scaled_mm_azp_sm75(torch::Tensor& c, torch::Tensor const& a,
@@ -36,24 +36,24 @@ void cutlass_scaled_mm_azp_sm75(torch::Tensor& c, torch::Tensor const& a,
36 torch::Tensor const& a_scales,
37 torch::Tensor const& b_scales,
38 torch::Tensor const& azp_adj,
39 - c10::optional<torch::Tensor> const& azp,
40 - c10::optional<torch::Tensor> const& bias);
39 + std::optional<torch::Tensor> const& azp,
40 + std::optional<torch::Tensor> const& bias);
41
42 void cutlass_scaled_mm_azp_sm80(torch::Tensor& c, torch::Tensor const& a,
43 torch::Tensor const& b,
44 torch::Tensor const& a_scales,
45 torch::Tensor const& b_scales,
46 torch::Tensor const& azp_adj,
47 - c10::optional<torch::Tensor> const& azp,
48 - c10::optional<torch::Tensor> const& bias);
47 + std::optional<torch::Tensor> const& azp,
48 + std::optional<torch::Tensor> const& bias);
49
50 void cutlass_scaled_mm_azp_sm89(torch::Tensor& c, torch::Tensor const& a,
51 torch::Tensor const& b,
52 torch::Tensor const& a_scales,
53 torch::Tensor const& b_scales,
54 torch::Tensor const& azp_adj,
55 - c10::optional<torch::Tensor> const& azp,
56 - c10::optional<torch::Tensor> const& bias);
55 + std::optional<torch::Tensor> const& azp,
56 + std::optional<torch::Tensor> const& bias);
57
58 #if defined CUDA_VERSION && CUDA_VERSION >= 12000
59 void cutlass_scaled_mm_azp_sm90(torch::Tensor& c, torch::Tensor const& a,
@@ -61,8 +61,8 @@ void cutlass_scaled_mm_azp_sm90(torch::Tensor& c, torch::Tensor const& a,
61 torch::Tensor const& a_scales,
62 torch::Tensor const& b_scales,
63 torch::Tensor const& azp_adj,
64 - c10::optional<torch::Tensor> const& azp,
65 - c10::optional<torch::Tensor> const& bias);
64 + std::optional<torch::Tensor> const& azp,
65 + std::optional<torch::Tensor> const& bias);
66 #endif
67
68 bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
@@ -84,7 +84,7 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
84 void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
85 torch::Tensor const& b, torch::Tensor const& a_scales,
86 torch::Tensor const& b_scales,
87 - c10::optional<torch::Tensor> const& bias) {
87 + std::optional<torch::Tensor> const& bias) {
88 // Checks for conformality
89 TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
90 TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
@@ -148,8 +148,8 @@ void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a,
148 torch::Tensor const& a_scales,
149 torch::Tensor const& b_scales,
150 torch::Tensor const& azp_adj,
151 - c10::optional<torch::Tensor> const& azp,
152 - c10::optional<torch::Tensor> const& bias) {
151 + std::optional<torch::Tensor> const& azp,
152 + std::optional<torch::Tensor> const& bias) {
153 // Checks for conformality
154 TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
155 TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
csrc/quantization/machete/generate.py
+1 -1
@@ -63,7 +63,7 @@ torch::Tensor mm_dispatch_{{type_sig}}(MMArgs args) {
63
64
65 static inline std::optional<at::ScalarType> maybe_scalartype(
66 - c10::optional<at::Tensor> const& t) {
66 + std::optional<at::Tensor> const& t) {
67 if (!t) {
68 return std::nullopt;
69 } else {
csrc/quantization/machete/machete_mm_kernel.cuh
+5 -5
@@ -183,11 +183,11 @@ struct MacheteKernelTemplate {
183 torch::Tensor const& A, // MxK matrix
184 torch::Tensor const& B, // KxN prepacked matrix
185 torch::Tensor& D, // MxN matrix
186 - c10::optional<torch::Tensor> const& maybe_g_scales, // scale_KxN matrix
187 - c10::optional<torch::Tensor> const& maybe_g_zeros, // scale_KxN matrix
188 - c10::optional<int64_t> maybe_group_size,
189 - c10::optional<torch::Tensor> const& maybe_ch_scales, // len N vector
190 - c10::optional<torch::Tensor> const& maybe_tok_scales) // len M vector
186 + std::optional<torch::Tensor> const& maybe_g_scales, // scale_KxN matrix
187 + std::optional<torch::Tensor> const& maybe_g_zeros, // scale_KxN matrix
188 + std::optional<int64_t> maybe_group_size,
189 + std::optional<torch::Tensor> const& maybe_ch_scales, // len N vector
190 + std::optional<torch::Tensor> const& maybe_tok_scales) // len M vector
191 {
192 static_assert(!with_group_zeropoints || with_group_scales);
193
csrc/quantization/machete/machete_mm_launcher.cuh
+12 -12
@@ -13,23 +13,23 @@ struct MMArgs {
13 torch::Tensor const& A;
14 torch::Tensor const& B;
15 vllm::ScalarType const& b_type;
16 - c10::optional<at::ScalarType> const& maybe_out_type;
17 - c10::optional<torch::Tensor> const& maybe_group_scales;
18 - c10::optional<torch::Tensor> const& maybe_group_zeros;
19 - c10::optional<int64_t> maybe_group_size;
20 - c10::optional<torch::Tensor> const& maybe_channel_scales;
21 - c10::optional<torch::Tensor> const& maybe_token_scales;
22 - c10::optional<std::string> maybe_schedule;
16 + std::optional<at::ScalarType> const& maybe_out_type;
17 + std::optional<torch::Tensor> const& maybe_group_scales;
18 + std::optional<torch::Tensor> const& maybe_group_zeros;
19 + std::optional<int64_t> maybe_group_size;
20 + std::optional<torch::Tensor> const& maybe_channel_scales;
21 + std::optional<torch::Tensor> const& maybe_token_scales;
22 + std::optional<std::string> maybe_schedule;
23 };
24
25 struct SupportedSchedulesArgs {
26 at::ScalarType a_type;
27 vllm::ScalarType b_type;
28 - c10::optional<at::ScalarType> maybe_group_scales_type;
29 - c10::optional<at::ScalarType> maybe_group_zeros_type;
30 - c10::optional<at::ScalarType> maybe_channel_scales_type;
31 - c10::optional<at::ScalarType> maybe_token_scales_type;
32 - c10::optional<at::ScalarType> maybe_out_type;
28 + std::optional<at::ScalarType> maybe_group_scales_type;
29 + std::optional<at::ScalarType> maybe_group_zeros_type;
30 + std::optional<at::ScalarType> maybe_channel_scales_type;
31 + std::optional<at::ScalarType> maybe_token_scales_type;
32 + std::optional<at::ScalarType> maybe_out_type;
33 };
34
35 torch::Tensor mm_dispatch(MMArgs args);
csrc/quantization/machete/machete_prepack_launcher.cuh
+1 -1
@@ -10,7 +10,7 @@ struct PrepackBArgs {
10 torch::Tensor const& B;
11 at::ScalarType a_type;
12 vllm::ScalarType b_type;
13 - c10::optional<at::ScalarType> maybe_group_scales_type;
13 + std::optional<at::ScalarType> maybe_group_scales_type;
14 };
15
16 template <typename PrepackedLayoutB>
csrc/quantization/machete/machete_pytorch.cu
+13 -13
@@ -10,11 +10,11 @@ using namespace vllm;
10
11 std::vector<std::string> supported_schedules(
12 at::ScalarType a_type, int64_t b_type_id,
13 - c10::optional<at::ScalarType> maybe_group_scales_type,
14 - c10::optional<at::ScalarType> maybe_group_zeros_type,
15 - c10::optional<at::ScalarType> maybe_channel_scales_type,
16 - c10::optional<at::ScalarType> maybe_token_scales_type,
17 - c10::optional<at::ScalarType> maybe_out_type) {
13 + std::optional<at::ScalarType> maybe_group_scales_type,
14 + std::optional<at::ScalarType> maybe_group_zeros_type,
15 + std::optional<at::ScalarType> maybe_channel_scales_type,
16 + std::optional<at::ScalarType> maybe_token_scales_type,
17 + std::optional<at::ScalarType> maybe_out_type) {
18 ScalarType const b_type = ScalarType::from_id(b_type_id);
19 return supported_schedules_dispatch({
20 .a_type = a_type,
@@ -29,13 +29,13 @@ std::vector<std::string> supported_schedules(
29
30 torch::Tensor mm(torch::Tensor const& A, torch::Tensor const& B,
31 int64_t b_type_id,
32 - c10::optional<at::ScalarType> const& maybe_out_type,
33 - c10::optional<torch::Tensor> const& maybe_group_scales,
34 - c10::optional<torch::Tensor> const& maybe_group_zeros,
35 - c10::optional<int64_t> maybe_group_size,
36 - c10::optional<torch::Tensor> const& maybe_channel_scales,
37 - c10::optional<torch::Tensor> const& maybe_token_scales,
38 - c10::optional<std::string> maybe_schedule) {
32 + std::optional<at::ScalarType> const& maybe_out_type,
33 + std::optional<torch::Tensor> const& maybe_group_scales,
34 + std::optional<torch::Tensor> const& maybe_group_zeros,
35 + std::optional<int64_t> maybe_group_size,
36 + std::optional<torch::Tensor> const& maybe_channel_scales,
37 + std::optional<torch::Tensor> const& maybe_token_scales,
38 + std::optional<std::string> maybe_schedule) {
39 ScalarType const b_type = ScalarType::from_id(b_type_id);
40 return mm_dispatch({.A = A,
41 .B = B,
@@ -51,7 +51,7 @@ torch::Tensor mm(torch::Tensor const& A, torch::Tensor const& B,
51
52 torch::Tensor prepack_B(
53 torch::Tensor const& B, at::ScalarType const& a_type, int64_t b_type_id,
54 - c10::optional<at::ScalarType> const& maybe_group_scales_type) {
54 + std::optional<at::ScalarType> const& maybe_group_scales_type) {
55 ScalarType const b_type = ScalarType::from_id(b_type_id);
56 return prepack_B_dispatch(
57 {.B = B,
csrc/rocm/attention.cu
+2 -2
@@ -928,7 +928,7 @@ void paged_attention_custom_launcher(
928 torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
929 torch::Tensor& value_cache, const int num_kv_heads, float scale,
930 torch::Tensor& block_tables, torch::Tensor& context_lens,
931 - int max_context_len, const c10::optional<torch::Tensor>& alibi_slopes,
931 + int max_context_len, const std::optional<torch::Tensor>& alibi_slopes,
932 float k_scale, float v_scale) {
933 int num_seqs = query.size(0);
934 int num_heads = query.size(1);
@@ -1086,7 +1086,7 @@ void paged_attention(
1086 torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
1087 torch::Tensor& context_lens, // [num_seqs]
1088 int64_t block_size, int64_t max_context_len,
1089 - const c10::optional<torch::Tensor>& alibi_slopes,
1089 + const std::optional<torch::Tensor>& alibi_slopes,
1090 const std::string& kv_cache_dtype, double k_scale, double v_scale) {
1091 const int head_size = query.size(2);
1092 if (kv_cache_dtype == "auto") {
csrc/rocm/ops.h
+1 -1
@@ -9,6 +9,6 @@ void paged_attention(torch::Tensor& out, torch::Tensor& exp_sums,
9 double scale, torch::Tensor& block_tables,
10 torch::Tensor& context_lens, int64_t block_size,
11 int64_t max_context_len,
12 - const c10::optional<torch::Tensor>& alibi_slopes,
12 + const std::optional<torch::Tensor>& alibi_slopes,
13 const std::string& kv_cache_dtype, double k_scale,
14 double v_scale);
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu
+1 -1
@@ -286,7 +286,7 @@ void cutlass_scaled_sparse_mm_sm90(torch::Tensor& out, torch::Tensor const& a,
286 torch::Tensor const& bt_meta,
287 torch::Tensor const& a_scales,
288 torch::Tensor const& b_scales,
289 - c10::optional<torch::Tensor> const& bias) {
289 + std::optional<torch::Tensor> const& bias) {
290 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
291 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
292 if (bias) {
csrc/sparse/cutlass/sparse_scaled_mm_entry.cu
+2 -2
@@ -22,7 +22,7 @@ void cutlass_scaled_sparse_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
22 torch::Tensor const& e,
23 torch::Tensor const& a_scales,
24 torch::Tensor const& b_scales,
25 - c10::optional<torch::Tensor> const& bias);
25 + std::optional<torch::Tensor> const& bias);
26 #endif
27
28 void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
@@ -30,7 +30,7 @@ void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
30 torch::Tensor const& bt_meta,
31 torch::Tensor const& a_scales,
32 torch::Tensor const& b_scales,
33 - c10::optional<torch::Tensor> const& bias) {
33 + std::optional<torch::Tensor> const& bias) {
34 // Checks for conformality
35 TORCH_CHECK(a.dim() == 2 && bt_nzs.dim() == 2 && c.dim() == 2);
36 TORCH_CHECK(c.size(1) == bt_nzs.size(0) && bt_nzs.size(1) * 2 == a.size(1) &&