@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
h 85 lines 3.96 KB
Raw
1 #pragma once
2
3 #include <torch/csrc/stable/tensor.h>
4
5 #include <optional>
6 #include <tuple>
7
8 void topk_softmax(torch::stable::Tensor& topk_weights,
9 torch::stable::Tensor& topk_indices,
10 torch::stable::Tensor& token_expert_indices,
11 torch::stable::Tensor& gating_output, bool renormalize,
12 std::optional<torch::stable::Tensor> bias,
13 std::optional<torch::stable::Tensor> is_padding);
14
15 void topk_sigmoid(torch::stable::Tensor& topk_weights,
16 torch::stable::Tensor& topk_indices,
17 torch::stable::Tensor& token_expert_indices,
18 torch::stable::Tensor& gating_output, bool renormalize,
19 std::optional<torch::stable::Tensor> bias,
20 double routed_scaling_factor,
21 std::optional<torch::stable::Tensor> is_padding);
22
23 void topk_softplus_sqrt(
24 torch::stable::Tensor& topk_weights, torch::stable::Tensor& topk_indices,
25 torch::stable::Tensor& token_expert_indices,
26 torch::stable::Tensor& gating_output, bool renormalize,
27 double routed_scaling_factor,
28 const std::optional<torch::stable::Tensor>& correction_bias,
29 const std::optional<torch::stable::Tensor>& input_ids,
30 const std::optional<torch::stable::Tensor>& tid2eid,
31 const std::optional<torch::stable::Tensor>& is_padding,
32 const std::optional<torch::stable::Tensor>& bias_vl,
33 int64_t image_sentinel_lo);
34
35 void moe_sum(torch::stable::Tensor& input, torch::stable::Tensor& output,
36 std::optional<torch::stable::Tensor> topk_ids,
37 std::optional<torch::stable::Tensor> expert_map);
38
39 void moe_align_block_size(torch::stable::Tensor topk_ids, int64_t num_experts,
40 int64_t block_size,
41 torch::stable::Tensor sorted_token_ids,
42 torch::stable::Tensor experts_ids,
43 torch::stable::Tensor num_tokens_post_pad,
44 std::optional<torch::stable::Tensor> maybe_expert_map,
45 std::optional<torch::stable::Tensor> scatter_idx);
46
47 void batched_moe_align_block_size(
48 int64_t max_tokens_per_batch, int64_t block_size,
49 const torch::stable::Tensor& expert_num_tokens,
50 torch::stable::Tensor sorted_ids, torch::stable::Tensor expert_ids,
51 torch::stable::Tensor num_tokens_post_pad);
52
53 void moe_lora_align_block_size(
54 torch::stable::Tensor topk_ids, torch::stable::Tensor token_lora_mapping,
55 int64_t num_experts, int64_t block_size, int64_t max_loras,
56 int64_t max_num_tokens_padded, int64_t max_num_m_blocks,
57 torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids,
58 torch::stable::Tensor num_tokens_post_pad,
59 torch::stable::Tensor adapter_enabled, torch::stable::Tensor lora_ids,
60 std::optional<torch::stable::Tensor> maybe_expert_map);
61 #ifndef USE_ROCM
62 torch::stable::Tensor moe_wna16_gemm(
63 torch::stable::Tensor input, torch::stable::Tensor output,
64 torch::stable::Tensor b_qweight, torch::stable::Tensor b_scales,
65 std::optional<torch::stable::Tensor> b_qzeros,
66 std::optional<torch::stable::Tensor> topk_weights,
67 torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids,
68 torch::stable::Tensor num_tokens_post_pad, int64_t top_k,
69 int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N, int64_t BLOCK_SIZE_K,
70 int64_t bit);
71
72 std::tuple<torch::stable::Tensor, torch::stable::Tensor> grouped_topk(
73 const torch::stable::Tensor& scores, int64_t n_group, int64_t topk_group,
74 int64_t topk, bool renormalize, double routed_scaling_factor,
75 const torch::stable::Tensor& bias, int64_t scoring_func);
76 #endif
77
78 bool moe_permute_unpermute_supported();
79
80 int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
81 int64_t num_expert);
82
83 void shuffle_rows(const torch::stable::Tensor& input_tensor,
84 const torch::stable::Tensor& dst2src_map,
85 torch::stable::Tensor& output_tensor);