A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 07:45 UTC
[PluggableLayer][MM] Add PluggableLayer for RelPosAttention (#33753)
Signed-off-by: shen-shanshan <467638484@qq.com>
Shanshan Shen committed
Mar 4, 2026 at 11:41 UTC
77e6dcbbfad8cfca6867663b164f038820f7a0be
2 files changed
+8
-1
docs/design/custom_op.md
+2
@@ -54,6 +54,8 @@ For example:
54
--8<-- "vllm/model_executor/layers/attention/mm_encoder_attention.py:mm_encoder_attn"
55
56
--8<-- "vllm/model_executor/layers/mla.py:multi_head_latent_attention"
57
+
58
+--8<-- "vllm/model_executor/models/deepencoder.py:rel_pos_attention"
59
```
60
61
**2. Activation:**
vllm/model_executor/models/deepencoder.py
+6
-1
@@ -18,6 +18,7 @@ import torch.nn as nn
18
import torch.nn.functional as F
19
from transformers import CLIPVisionConfig
20
21
+from vllm.model_executor.custom_op import PluggableLayer
22
from vllm.model_executor.layers.attention import MMEncoderAttention
23
from vllm.model_executor.layers.conv import Conv2dLayer
24
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -263,9 +264,13 @@ class Block(nn.Module):
264
return x
265
266
266
-class RelPosAttention(nn.Module):
267
+# --8<-- [start:rel_pos_attention]
268
+@PluggableLayer.register("rel_pos_attention")
269
+class RelPosAttention(PluggableLayer):
270
"""Multi-head Attention block with relative position embeddings."""
271
272
+ # --8<-- [end:rel_pos_attention]
273
+
274
def __init__(
275
self,
276
dim: int,