@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 76ed5340f0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] add deepseek v2 compile (#9775)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Oct 28, 2024 at 14:35 UTC 76ed5340f0ec0e481593ea1a94459b4b55136a4f
1 file changed +2
vllm/model_executor/models/deepseek_v2.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28 from transformers import PretrainedConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 +from vllm.compilation.decorators import support_torch_compile
32 from vllm.config import CacheConfig
33 from vllm.distributed import (get_pp_group,
34 get_tensor_model_parallel_world_size,
@@ -403,6 +404,7 @@ class DeepseekV2DecoderLayer(nn.Module):
404 return hidden_states, residual
405
406
407 +@support_torch_compile
408 class DeepseekV2Model(nn.Module):
409
410 fall_back_to_pt_during_load = False