@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 571841b7fc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[torch.compile] support encoder based models (#10613)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 24, 2024 at 21:24 UTC 571841b7fcc67f8b1d171522f6249ed4224033e1
2 files changed +17 -10
tests/compile/test_basic_correctness.py
+10
@@ -62,6 +62,16 @@ test_settings = [
62 method="encode",
63 fullgraph=True,
64 ),
65 + # encoder-based embedding model (BERT)
66 + TestSetting(
67 + model="BAAI/bge-base-en-v1.5",
68 + model_args=["--task", "embedding"],
69 + pp_size=1,
70 + tp_size=1,
71 + attn_backend="XFORMERS",
72 + method="encode",
73 + fullgraph=True,
74 + ),
75 # vision language model
76 TestSetting(
77 model="microsoft/Phi-3.5-vision-instruct",
vllm/model_executor/models/bert.py
+7 -10
@@ -5,6 +5,7 @@ from torch import nn
5 from transformers import BertConfig
6
7 from vllm.attention import Attention, AttentionMetadata, AttentionType
8 +from vllm.compilation.decorators import support_torch_compile
9 from vllm.config import CacheConfig, PoolerConfig, VllmConfig
10 from vllm.distributed import get_tensor_model_parallel_world_size
11 from vllm.model_executor.layers.activation import get_act_fn
@@ -92,14 +93,14 @@ class BertPooler(nn.Module):
93 return pooled_output
94
95
96 +@support_torch_compile
97 class BertEncoder(nn.Module):
98
97 - def __init__(self,
98 - config: BertConfig,
99 - cache_config: Optional[CacheConfig] = None,
100 - quant_config: Optional[QuantizationConfig] = None,
101 - prefix: str = ""):
99 + def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
100 super().__init__()
101 + config = vllm_config.model_config.hf_config
102 + cache_config = vllm_config.cache_config
103 + quant_config = vllm_config.quant_config
104 self.layer = nn.ModuleList([
105 BertLayer(config=config,
106 cache_config=cache_config,
@@ -336,12 +337,8 @@ class BertModel(nn.Module):
337 add_pooling_layer: bool = False):
338 super().__init__()
339 config = vllm_config.model_config.hf_config
339 - cache_config = vllm_config.cache_config
340 - quant_config = vllm_config.quant_config
340 self.embeddings = embedding_class(config)
342 - self.encoder = BertEncoder(config,
343 - cache_config,
344 - quant_config,
341 + self.encoder = BertEncoder(vllm_config=vllm_config,
342 prefix=f"{prefix}.encoder")
343 self.pooler = BertPooler(config) if add_pooling_layer else None
344