A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Model] Adding torch compile annotations to chatglm (#15624)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Mar 28, 2025 at 21:14 UTC
91276c57210b36997861af706a48ac784573ed4c
1 file changed
+2
vllm/model_executor/models/chatglm.py
+2
@@ -10,6 +10,7 @@ from torch import nn
10
from torch.nn import LayerNorm
11
12
from vllm.attention import Attention
13
+from vllm.compilation.decorators import support_torch_compile
14
from vllm.config import CacheConfig, VllmConfig
15
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
16
from vllm.model_executor.layers.activation import SiluAndMul
@@ -293,6 +294,7 @@ class GLMTransformer(nn.Module):
294
return hidden_states
295
296
297
+@support_torch_compile
298
class ChatGLMModel(nn.Module):
299
300
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):