@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3194039c0e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

Apply torch.compile to fused_moe/grouped_topk (#12637)

Michael Goin committed Feb 1, 2025 at 11:16 UTC 3194039c0ee82685af434c9f8023304b4a45124b
2 files changed +3 -2
vllm/model_executor/layers/fused_moe/fused_moe.py
+1
@@ -759,6 +759,7 @@ def fused_topk(
759
760
761 # This is used by the Deepseek-V2 and Deepseek-V3 model
762 +@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
763 def grouped_topk(hidden_states: torch.Tensor,
764 gating_output: torch.Tensor,
765 topk: int,
vllm/model_executor/models/deepseek_v3.py
+2 -2
@@ -27,6 +27,7 @@ from torch import nn
27 from transformers import PretrainedConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 +from vllm.compilation.decorators import support_torch_compile
31 from vllm.config import CacheConfig, ModelConfig, VllmConfig
32 from vllm.distributed import (get_pp_group,
33 get_tensor_model_parallel_world_size,
@@ -566,8 +567,7 @@ class DeepseekV3DecoderLayer(nn.Module):
567 return hidden_states, residual
568
569
569 -# TODO(simon): check whether we support torch compile for Deepseek V3
570 -# @support_torch_compile
570 +@support_torch_compile
571 class DeepseekV3Model(nn.Module):
572
573 fall_back_to_pt_during_load = False