A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Easy] Eliminate c10::optional usage in vllm/csrc (#17819)
Lu Fang committed
May 8, 2025 at 03:05 UTC
f50dcb7c215bf5468727839ebd1a413e6bd49387
4 files changed
+7
-7
csrc/quantization/gptq_allspark/allspark_qgemm_w8a16.cu
+2
-2
@@ -9,7 +9,7 @@ at::Tensor as_g_workspace;
9
10
torch::Tensor allspark_w8a16_gemm(
11
torch::Tensor const& a, torch::Tensor const& b_qweight,
12
- torch::Tensor const& b_scales, c10::optional<torch::Tensor> const& b_qzeros,
12
+ torch::Tensor const& b_scales, std::optional<torch::Tensor> const& b_qzeros,
13
int64_t n, int64_t group_size, int64_t sm_count, int64_t sm_version,
14
int64_t CUBLAS_M_THRESHOLD, bool has_zp, bool n32k16_reorder) {
15
TORCH_CHECK_NOT_IMPLEMENTED(
@@ -918,7 +918,7 @@ void allspark_qgemm_w8a16_perc_ampere(
918
919
torch::Tensor allspark_w8a16_gemm(
920
torch::Tensor const& a, torch::Tensor const& b_qweight,
921
- torch::Tensor const& b_scales, c10::optional<torch::Tensor> const& b_qzeros,
921
+ torch::Tensor const& b_scales, std::optional<torch::Tensor> const& b_qzeros,
922
int64_t n, int64_t group_size, int64_t sm_count, int64_t sm_version,
923
int64_t CUBLAS_M_THRESHOLD, bool has_zp, bool n32k16_reorder) {
924
// Verify device and strides
csrc/quantization/gptq_allspark/allspark_repack.cu
+2
-2
@@ -100,9 +100,9 @@ void rearrange_kn_weight_as_n32k16_order_ldg16(
100
101
void rearrange_kn_weight_as_n32k16_order(
102
torch::Tensor const& b_qweight, torch::Tensor const& b_scales,
103
- c10::optional<torch::Tensor> const& b_zeros, bool has_zp,
103
+ std::optional<torch::Tensor> const& b_zeros, bool has_zp,
104
torch::Tensor& b_qweight_reorder, torch::Tensor& b_scales_reorder,
105
- c10::optional<torch::Tensor> const& b_zeros_reorder, const int64_t K,
105
+ std::optional<torch::Tensor> const& b_zeros_reorder, const int64_t K,
106
const int64_t N, const int64_t N_32align) {
107
// Verify device and strides
108
TORCH_CHECK(b_qweight.device().is_cuda(), "b_qweight is not on GPU");
csrc/rocm/attention.cu
+2
-2
@@ -1597,7 +1597,7 @@ void paged_attention_custom_launcher(
1597
torch::Tensor& block_tables, torch::Tensor& context_lens,
1598
const std::optional<torch::Tensor>& query_start_loc, int max_context_len,
1599
const std::optional<torch::Tensor>& alibi_slopes, torch::Tensor& k_scale,
1600
- torch::Tensor& v_scale, const c10::optional<torch::Tensor>& fp8_out_scale) {
1600
+ torch::Tensor& v_scale, const std::optional<torch::Tensor>& fp8_out_scale) {
1601
int num_seqs = block_tables.size(0);
1602
int num_heads = query.size(1);
1603
int head_size = query.size(2);
@@ -1825,7 +1825,7 @@ void paged_attention(
1825
const std::optional<torch::Tensor>& alibi_slopes,
1826
const std::string& kv_cache_dtype, torch::Tensor& k_scale,
1827
torch::Tensor& v_scale,
1828
- const c10::optional<torch::Tensor>& fp8_out_scale) {
1828
+ const std::optional<torch::Tensor>& fp8_out_scale) {
1829
// clang-format on
1830
const int head_size = query.size(2);
1831
if (kv_cache_dtype == "auto") {
csrc/rocm/ops.h
+1
-1
@@ -19,4 +19,4 @@ void paged_attention(
19
const std::optional<torch::Tensor>& query_start_loc, int64_t block_size,
20
int64_t max_context_len, const std::optional<torch::Tensor>& alibi_slopes,
21
const std::string& kv_cache_dtype, torch::Tensor& k_scale,
22
- torch::Tensor& v_scale, const c10::optional<torch::Tensor>& fp8_out_scale);
22
+ torch::Tensor& v_scale, const std::optional<torch::Tensor>& fp8_out_scale);