A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] add deepseek v2 compile (#9775)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Oct 28, 2024 at 14:35 UTC
76ed5340f0ec0e481593ea1a94459b4b55136a4f
1 file changed
+2
vllm/model_executor/models/deepseek_v2.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28
from transformers import PretrainedConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
+from vllm.compilation.decorators import support_torch_compile
32
from vllm.config import CacheConfig
33
from vllm.distributed import (get_pp_group,
34
get_tensor_model_parallel_world_size,
@@ -403,6 +404,7 @@ class DeepseekV2DecoderLayer(nn.Module):
404
return hidden_states, residual
405
406
407
+@support_torch_compile
408
class DeepseekV2Model(nn.Module):
409
410
fall_back_to_pt_during_load = False