A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[torch.compile] support encoder based models (#10613)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 24, 2024 at 21:24 UTC
571841b7fcc67f8b1d171522f6249ed4224033e1
2 files changed
+17
-10
tests/compile/test_basic_correctness.py
+10
@@ -62,6 +62,16 @@ test_settings = [
62
method="encode",
63
fullgraph=True,
64
),
65
+ # encoder-based embedding model (BERT)
66
+ TestSetting(
67
+ model="BAAI/bge-base-en-v1.5",
68
+ model_args=["--task", "embedding"],
69
+ pp_size=1,
70
+ tp_size=1,
71
+ attn_backend="XFORMERS",
72
+ method="encode",
73
+ fullgraph=True,
74
+ ),
75
# vision language model
76
TestSetting(
77
model="microsoft/Phi-3.5-vision-instruct",
vllm/model_executor/models/bert.py
+7
-10
@@ -5,6 +5,7 @@ from torch import nn
5
from transformers import BertConfig
6
7
from vllm.attention import Attention, AttentionMetadata, AttentionType
8
+from vllm.compilation.decorators import support_torch_compile
9
from vllm.config import CacheConfig, PoolerConfig, VllmConfig
10
from vllm.distributed import get_tensor_model_parallel_world_size
11
from vllm.model_executor.layers.activation import get_act_fn
@@ -92,14 +93,14 @@ class BertPooler(nn.Module):
93
return pooled_output
94
95
96
+@support_torch_compile
97
class BertEncoder(nn.Module):
98
97
- def __init__(self,
98
- config: BertConfig,
99
- cache_config: Optional[CacheConfig] = None,
100
- quant_config: Optional[QuantizationConfig] = None,
101
- prefix: str = ""):
99
+ def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
100
super().__init__()
101
+ config = vllm_config.model_config.hf_config
102
+ cache_config = vllm_config.cache_config
103
+ quant_config = vllm_config.quant_config
104
self.layer = nn.ModuleList([
105
BertLayer(config=config,
106
cache_config=cache_config,
@@ -336,12 +337,8 @@ class BertModel(nn.Module):
337
add_pooling_layer: bool = False):
338
super().__init__()
339
config = vllm_config.model_config.hf_config
339
- cache_config = vllm_config.cache_config
340
- quant_config = vllm_config.quant_config
340
self.embeddings = embedding_class(config)
342
- self.encoder = BertEncoder(config,
343
- cache_config,
344
- quant_config,
341
+ self.encoder = BertEncoder(vllm_config=vllm_config,
342
prefix=f"{prefix}.encoder")
343
self.pooler = BertPooler(config) if add_pooling_layer else None
344