A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix / Core] Prefix Caching Guards (merged with main) (#4846)
Co-authored-by: rsnm2 <rshaw@neuralmagic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com>
Zhuohan Li committed
May 27, 2024 at 15:18 UTC
1102bef2195a102c6a5489a28329e543a600b4d8
11 files changed
+167
-44
tests/prefix_caching/test_disable_sliding_window.py
new
+44
@@ -0,0 +1,44 @@
1
+"""Compare the with and without prefix caching.
2
+
3
+Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4
+"""
5
+import pytest
6
+
7
+from tests.conftest import cleanup
8
+from vllm import LLM
9
+
10
+MODEL_LEN_LEN = [
11
+ # Example models with sliding window.
12
+ ("bigcode/starcoder2-3b", 4096, 16384),
13
+ # ("mistralai/Mistral-7B-v0.1", 4096, 32768), << OOM in CI
14
+
15
+ # Confirm model with sliding window works.
16
+ # config has "use_sliding_window": false
17
+ ("Qwen/Qwen1.5-0.5B-Chat", 32768, 32768),
18
+ # config has no sliding window attribute.
19
+ ("TinyLlama/TinyLlama-1.1B-Chat-v1.0", 2048, 2048),
20
+]
21
+
22
+
23
+@pytest.mark.parametrize("model_len_len", MODEL_LEN_LEN)
24
+def test_disable_sliding_window(model_len_len, ):
25
+ model, sliding_len, full_len = model_len_len
26
+ vllm_disabled_model = LLM(model, disable_sliding_window=True)
27
+ vllm_disabled_model.generate("Hi my name is")
28
+ model_config = vllm_disabled_model.llm_engine.model_config
29
+ assert model_config.max_model_len == sliding_len, (
30
+ "Max len expected to equal sliding_len of %s, but got %s", sliding_len,
31
+ model_config.max_model_len)
32
+
33
+ del vllm_disabled_model
34
+ cleanup()
35
+
36
+ vllm_enabled_model = LLM(model, disable_sliding_window=False)
37
+ vllm_enabled_model.generate("Hi my name is")
38
+ model_config = vllm_enabled_model.llm_engine.model_config
39
+ assert model_config.max_model_len == full_len, (
40
+ "Max len expected to equal full_len of %s, but got %s", full_len,
41
+ model_config.max_model_len)
42
+
43
+ del vllm_enabled_model
44
+ cleanup()
tests/test_config.py
+24
@@ -1,5 +1,29 @@
1
+import pytest
2
+
3
from vllm.config import ModelConfig
4
5
+MODEL_IDS_EXPECTED = [
6
+ ("Qwen/Qwen1.5-7B", 32768),
7
+ ("mistralai/Mistral-7B-v0.1", 4096),
8
+ ("mistralai/Mistral-7B-Instruct-v0.2", 32768),
9
+]
10
+
11
+
12
+@pytest.mark.parametrize("model_id_expected", MODEL_IDS_EXPECTED)
13
+def test_disable_sliding_window(model_id_expected):
14
+ model_id, expected = model_id_expected
15
+ model_config = ModelConfig(
16
+ model_id,
17
+ model_id,
18
+ tokenizer_mode="auto",
19
+ trust_remote_code=False,
20
+ seed=0,
21
+ dtype="float16",
22
+ revision=None,
23
+ disable_sliding_window=True,
24
+ )
25
+ assert model_config.max_model_len == expected
26
+
27
28
def test_get_sliding_window():
29
TEST_SLIDING_WINDOW = 4096
vllm/attention/layer.py
+2
-1
@@ -30,7 +30,6 @@ class Attention(nn.Module):
30
scale: float,
31
num_kv_heads: Optional[int] = None,
32
alibi_slopes: Optional[List[float]] = None,
33
- sliding_window: Optional[int] = None,
33
cache_config: Optional[CacheConfig] = None,
34
quant_config: Optional[QuantizationConfig] = None,
35
blocksparse_params: Optional[Dict[str, Any]] = None,
@@ -39,9 +38,11 @@ class Attention(nn.Module):
38
if cache_config is not None:
39
kv_cache_dtype = cache_config.cache_dtype
40
block_size = cache_config.block_size
41
+ sliding_window = cache_config.sliding_window
42
else:
43
kv_cache_dtype = "auto"
44
block_size = 16
45
+ sliding_window = None
46
if num_kv_heads is None:
47
num_kv_heads = num_heads
48
vllm/config.py
+64
-3
@@ -69,6 +69,10 @@ class ModelConfig:
69
max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
70
When a sequence has context length larger than this, we fall back
71
to eager mode
72
+ disable_sliding_window: Whether to disable sliding window. If True,
73
+ we will disable the sliding window functionality of the model.
74
+ If the model does not support sliding window, this argument is
75
+ ignored.
76
skip_tokenizer_init: If true, skip initialization of tokenizer and
77
detokenizer.
78
served_model_name: The model name used in metrics tag `model_name`,
@@ -96,6 +100,7 @@ class ModelConfig:
100
max_context_len_to_capture: Optional[int] = None,
101
max_seq_len_to_capture: Optional[int] = None,
102
max_logprobs: int = 5,
103
+ disable_sliding_window: bool = False,
104
skip_tokenizer_init: bool = False,
105
served_model_name: Optional[Union[str, List[str]]] = None,
106
) -> None:
@@ -118,14 +123,18 @@ class ModelConfig:
123
self.max_seq_len_to_capture = (max_seq_len_to_capture
124
or max_context_len_to_capture)
125
self.max_logprobs = max_logprobs
126
+ self.disable_sliding_window = disable_sliding_window
127
self.skip_tokenizer_init = skip_tokenizer_init
128
129
self.hf_config = get_config(self.model, trust_remote_code, revision,
130
code_revision, rope_scaling)
131
self.hf_text_config = get_hf_text_config(self.hf_config)
132
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
127
- self.max_model_len = _get_and_verify_max_len(self.hf_text_config,
128
- max_model_len)
133
+ self.max_model_len = _get_and_verify_max_len(
134
+ hf_config=self.hf_text_config,
135
+ max_model_len=max_model_len,
136
+ disable_sliding_window=self.disable_sliding_window,
137
+ sliding_window_len=self.get_hf_config_sliding_window())
138
self.served_model_name = get_served_model_name(model,
139
served_model_name)
140
if not self.skip_tokenizer_init:
@@ -220,7 +229,7 @@ class ModelConfig:
229
"must be divisible by pipeline parallel size "
230
f"({pipeline_parallel_size}).")
231
223
- def get_sliding_window(self) -> Optional[int]:
232
+ def get_hf_config_sliding_window(self) -> Optional[int]:
233
"""Get the sliding window size, or None if disabled.
234
"""
235
@@ -232,6 +241,15 @@ class ModelConfig:
241
return None
242
return getattr(self.hf_text_config, "sliding_window", None)
243
244
+ def get_sliding_window(self) -> Optional[int]:
245
+ """Get the sliding window size, or None if disabled.
246
+ """
247
+ # If user disables sliding window, return None.
248
+ if self.disable_sliding_window:
249
+ return None
250
+ # Otherwise get the value from the hf config.
251
+ return self.get_hf_config_sliding_window()
252
+
253
def get_vocab_size(self) -> int:
254
return self.hf_text_config.vocab_size
255
@@ -336,6 +354,7 @@ class CacheConfig:
354
self.enable_prefix_caching = enable_prefix_caching
355
self._verify_args()
356
self._verify_cache_dtype()
357
+ self._verify_prefix_caching()
358
359
# Will be set after profiling.
360
self.num_gpu_blocks = None
@@ -364,6 +383,19 @@ class CacheConfig:
383
else:
384
raise ValueError(f"Unknown kv cache dtype: {self.cache_dtype}")
385
386
+ def _verify_prefix_caching(self) -> None:
387
+ if not self.enable_prefix_caching:
388
+ return
389
+
390
+ if self.sliding_window is not None:
391
+ raise NotImplementedError(
392
+ "Prefix caching is not supported with sliding window. "
393
+ "Run with --disable-sliding-window to use prefix caching.")
394
+ if self.cache_dtype == "fp8":
395
+ raise NotImplementedError(
396
+ "Prefix caching is not supported for fp8 cache_dtype. "
397
+ "Run with --kv-cache-dtype auto to use prefix caching.")
398
+
399
def verify_with_parallel_config(
400
self,
401
parallel_config: "ParallelConfig",
@@ -1116,6 +1148,8 @@ def _get_and_verify_dtype(
1148
def _get_and_verify_max_len(
1149
hf_config: PretrainedConfig,
1150
max_model_len: Optional[int],
1151
+ disable_sliding_window: bool,
1152
+ sliding_window_len: Optional[int],
1153
) -> int:
1154
"""Get and verify the model's maximum length."""
1155
derived_max_model_len = float("inf")
@@ -1135,6 +1169,7 @@ def _get_and_verify_max_len(
1169
"max_seq_length",
1170
"seq_len",
1171
]
1172
+ # Choose the smallest "max_length" from the possible keys.
1173
max_len_key = None
1174
for key in possible_keys:
1175
max_len = getattr(hf_config, key, None)
@@ -1142,6 +1177,16 @@ def _get_and_verify_max_len(
1177
max_len_key = key if max_len < derived_max_model_len \
1178
else max_len_key
1179
derived_max_model_len = min(derived_max_model_len, max_len)
1180
+
1181
+ # If sliding window is manually disabled, max_length should be less
1182
+ # than the sliding window length in the model config.
1183
+ if disable_sliding_window and sliding_window_len is not None:
1184
+ max_len_key = "sliding_window" \
1185
+ if sliding_window_len < derived_max_model_len else max_len_key
1186
+ derived_max_model_len = min(derived_max_model_len, sliding_window_len)
1187
+
1188
+ # If none of the keys were found in the config, use a default and
1189
+ # log a warning.
1190
if derived_max_model_len == float("inf"):
1191
if max_model_len is not None:
1192
# If max_model_len is specified, we use it.
@@ -1157,6 +1202,13 @@ def _get_and_verify_max_len(
1202
1203
rope_scaling = getattr(hf_config, "rope_scaling", None)
1204
if rope_scaling is not None and rope_scaling["type"] != "su":
1205
+ if disable_sliding_window:
1206
+ # TODO(robertgshaw): Find a model that supports rope_scaling
1207
+ # with sliding window to see if this case should be allowed.
1208
+ raise NotImplementedError(
1209
+ "Disabling sliding window is not supported for models "
1210
+ "with rope_scaling. Please raise an issue so we can "
1211
+ "investigate.")
1212
assert "factor" in rope_scaling
1213
scaling_factor = rope_scaling["factor"]
1214
if rope_scaling["type"] == "yarn":
@@ -1164,6 +1216,8 @@ def _get_and_verify_max_len(
1216
"original_max_position_embeddings"]
1217
derived_max_model_len *= scaling_factor
1218
1219
+ # If the user specified a max length, make sure it is smaller than the
1220
+ # derived length from the HF model config.
1221
if max_model_len is None:
1222
max_model_len = int(derived_max_model_len)
1223
elif max_model_len > derived_max_model_len:
@@ -1172,6 +1226,13 @@ def _get_and_verify_max_len(
1226
# with model_max_length and allow this override when it's smaller.
1227
model_max_length = getattr(hf_config, "model_max_length", None)
1228
if model_max_length is not None and max_model_len <= model_max_length:
1229
+ if disable_sliding_window:
1230
+ # TODO(robertgshaw): Find a model that has model_max_length
1231
+ # with sliding window to see if this case should be allowed.
1232
+ raise NotImplementedError(
1233
+ "Disabling sliding window is not supported for models "
1234
+ "model_max_length in the config. Please raise an issue "
1235
+ "so we can investigate.")
1236
pass
1237
else:
1238
raise ValueError(
vllm/engine/arg_utils.py
+9
-3
@@ -41,6 +41,7 @@ class EngineArgs:
41
max_parallel_loading_workers: Optional[int] = None
42
block_size: int = 16
43
enable_prefix_caching: bool = False
44
+ disable_sliding_window: bool = False
45
use_v2_block_manager: bool = False
46
swap_space: int = 4 # GiB
47
gpu_memory_utilization: float = 0.90
@@ -267,6 +268,10 @@ class EngineArgs:
268
parser.add_argument('--enable-prefix-caching',
269
action='store_true',
270
help='Enables automatic prefix caching.')
271
+ parser.add_argument('--disable-sliding-window',
272
+ action='store_true',
273
+ help='Disables sliding window, '
274
+ 'capping to sliding window size')
275
parser.add_argument('--use-v2-block-manager',
276
action='store_true',
277
help='Use BlockSpaceMangerV2.')
@@ -558,8 +563,8 @@ class EngineArgs:
563
self.max_model_len, self.quantization,
564
self.quantization_param_path, self.enforce_eager,
565
self.max_context_len_to_capture, self.max_seq_len_to_capture,
561
- self.max_logprobs, self.skip_tokenizer_init,
562
- self.served_model_name)
566
+ self.max_logprobs, self.disable_sliding_window,
567
+ self.skip_tokenizer_init, self.served_model_name)
568
cache_config = CacheConfig(self.block_size,
569
self.gpu_memory_utilization,
570
self.swap_space, self.kv_cache_dtype,
@@ -645,7 +650,8 @@ class EngineArgs:
650
if (model_config.get_sliding_window() is not None
651
and scheduler_config.chunked_prefill_enabled):
652
raise ValueError(
648
- "Chunked prefill is not supported with sliding window.")
653
+ "Chunked prefill is not supported with sliding window. "
654
+ "Set --disable-sliding-window to disable sliding window.")
655
656
return EngineConfig(model_config=model_config,
657
cache_config=cache_config,
vllm/model_executor/models/llama.py
-4
@@ -94,7 +94,6 @@ class LlamaAttention(nn.Module):
94
max_position_embeddings: int = 8192,
95
quant_config: Optional[QuantizationConfig] = None,
96
bias: bool = False,
97
- sliding_window: Optional[int] = None,
97
cache_config: Optional[CacheConfig] = None,
98
) -> None:
99
super().__init__()
@@ -146,7 +145,6 @@ class LlamaAttention(nn.Module):
145
self.head_dim,
146
self.scaling,
147
num_kv_heads=self.num_kv_heads,
149
- sliding_window=sliding_window,
148
cache_config=cache_config,
149
quant_config=quant_config)
150
@@ -183,7 +181,6 @@ class LlamaDecoderLayer(nn.Module):
181
config.original_max_position_embeddings)
182
max_position_embeddings = getattr(config, "max_position_embeddings",
183
8192)
186
- sliding_window = getattr(config, "sliding_window", None)
184
# Support abacusai/Smaug-72B-v0.1 with attention_bias
185
# Support internlm/internlm-7b with bias
186
attention_bias = getattr(config, "attention_bias", False) or getattr(
@@ -198,7 +195,6 @@ class LlamaDecoderLayer(nn.Module):
195
max_position_embeddings=max_position_embeddings,
196
quant_config=quant_config,
197
bias=attention_bias,
201
- sliding_window=sliding_window,
198
cache_config=cache_config,
199
)
200
self.mlp = LlamaMLP(
vllm/model_executor/models/mixtral.py
+10
-12
@@ -246,15 +246,16 @@ class MixtralMoE(nn.Module):
246
247
class MixtralAttention(nn.Module):
248
249
- def __init__(self,
250
- hidden_size: int,
251
- num_heads: int,
252
- num_kv_heads: int,
253
- max_position: int = 4096 * 32,
254
- rope_theta: float = 10000,
255
- cache_config: Optional[CacheConfig] = None,
256
- quant_config: Optional[QuantizationConfig] = None,
257
- sliding_window: Optional[int] = None) -> None:
249
+ def __init__(
250
+ self,
251
+ hidden_size: int,
252
+ num_heads: int,
253
+ num_kv_heads: int,
254
+ max_position: int = 4096 * 32,
255
+ rope_theta: float = 10000,
256
+ cache_config: Optional[CacheConfig] = None,
257
+ quant_config: Optional[QuantizationConfig] = None,
258
+ ) -> None:
259
super().__init__()
260
self.hidden_size = hidden_size
261
tp_size = get_tensor_model_parallel_world_size()
@@ -276,7 +277,6 @@ class MixtralAttention(nn.Module):
277
self.kv_size = self.num_kv_heads * self.head_dim
278
self.scaling = self.head_dim**-0.5
279
self.rope_theta = rope_theta
279
- self.sliding_window = sliding_window
280
281
if isinstance(
282
quant_config,
@@ -312,7 +312,6 @@ class MixtralAttention(nn.Module):
312
self.head_dim,
313
self.scaling,
314
num_kv_heads=self.num_kv_heads,
315
- sliding_window=self.sliding_window,
315
cache_config=cache_config,
316
quant_config=quant_config)
317
@@ -349,7 +348,6 @@ class MixtralDecoderLayer(nn.Module):
348
max_position=config.max_position_embeddings,
349
num_kv_heads=config.num_key_value_heads,
350
rope_theta=rope_theta,
352
- sliding_window=config.sliding_window,
351
cache_config=cache_config,
352
quant_config=quant_config)
353
self.block_sparse_moe = MixtralMoE(
vllm/model_executor/models/mixtral_quant.py
-4
@@ -166,7 +166,6 @@ class MixtralAttention(nn.Module):
166
max_position: int = 4096 * 32,
167
rope_theta: float = 10000,
168
quant_config: Optional[QuantizationConfig] = None,
169
- sliding_window: Optional[int] = None,
169
cache_config: Optional[CacheConfig] = None,
170
) -> None:
171
super().__init__()
@@ -190,7 +189,6 @@ class MixtralAttention(nn.Module):
189
self.kv_size = self.num_kv_heads * self.head_dim
190
self.scaling = self.head_dim**-0.5
191
self.rope_theta = rope_theta
193
- self.sliding_window = sliding_window
192
193
self.qkv_proj = QKVParallelLinear(
194
hidden_size,
@@ -217,7 +215,6 @@ class MixtralAttention(nn.Module):
215
self.head_dim,
216
self.scaling,
217
num_kv_heads=self.num_kv_heads,
220
- sliding_window=self.sliding_window,
218
cache_config=cache_config,
219
quant_config=quant_config)
220
@@ -254,7 +251,6 @@ class MixtralDecoderLayer(nn.Module):
251
max_position=config.max_position_embeddings,
252
num_kv_heads=config.num_key_value_heads,
253
rope_theta=rope_theta,
257
- sliding_window=config.sliding_window,
254
cache_config=cache_config,
255
quant_config=quant_config)
256
self.block_sparse_moe = MixtralMoE(config=config,
vllm/model_executor/models/qwen2.py
+14
-11
@@ -86,10 +86,8 @@ class Qwen2Attention(nn.Module):
86
num_kv_heads: int,
87
max_position: int = 4096 * 32,
88
rope_theta: float = 10000,
89
- use_sliding_window: bool = False,
89
cache_config: Optional[CacheConfig] = None,
90
quant_config: Optional[QuantizationConfig] = None,
92
- sliding_window: Optional[int] = None,
91
rope_scaling: Optional[Tuple] = None) -> None:
92
super().__init__()
93
self.hidden_size = hidden_size
@@ -112,7 +110,6 @@ class Qwen2Attention(nn.Module):
110
self.kv_size = self.num_kv_heads * self.head_dim
111
self.scaling = self.head_dim**-0.5
112
self.rope_theta = rope_theta
115
- self.sliding_window = sliding_window if use_sliding_window else None
113
114
self.qkv_proj = QKVParallelLinear(
115
hidden_size,
@@ -140,7 +137,6 @@ class Qwen2Attention(nn.Module):
137
self.head_dim,
138
self.scaling,
139
num_kv_heads=self.num_kv_heads,
143
- sliding_window=self.sliding_window,
140
cache_config=cache_config,
141
quant_config=quant_config)
142
@@ -164,7 +160,6 @@ class Qwen2DecoderLayer(nn.Module):
160
def __init__(
161
self,
162
config: Qwen2Config,
167
- layer_idx: int,
163
cache_config: Optional[CacheConfig] = None,
164
quant_config: Optional[QuantizationConfig] = None,
165
) -> None:
@@ -173,18 +168,14 @@ class Qwen2DecoderLayer(nn.Module):
168
# Requires transformers > 4.32.0
169
rope_theta = getattr(config, "rope_theta", 1000000)
170
rope_scaling = getattr(config, "rope_scaling", None)
176
- use_sliding_window = (config.use_sliding_window
177
- and layer_idx < config.max_window_layers)
171
self.self_attn = Qwen2Attention(
172
hidden_size=self.hidden_size,
173
num_heads=config.num_attention_heads,
174
max_position=config.max_position_embeddings,
175
num_kv_heads=config.num_key_value_heads,
176
rope_theta=rope_theta,
184
- use_sliding_window=use_sliding_window,
177
cache_config=cache_config,
178
quant_config=quant_config,
187
- sliding_window=config.sliding_window,
179
rope_scaling=rope_scaling)
180
self.mlp = Qwen2MLP(
181
hidden_size=self.hidden_size,
@@ -244,8 +235,8 @@ class Qwen2Model(nn.Module):
235
config.hidden_size,
236
)
237
self.layers = nn.ModuleList([
247
- Qwen2DecoderLayer(config, layer_idx, cache_config, quant_config)
248
- for layer_idx in range(config.num_hidden_layers)
238
+ Qwen2DecoderLayer(config, cache_config, quant_config)
239
+ for _ in range(config.num_hidden_layers)
240
])
241
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
242
@@ -302,6 +293,18 @@ class Qwen2ForCausalLM(nn.Module):
293
lora_config: Optional[LoRAConfig] = None,
294
) -> None:
295
del lora_config
296
+ # TODO (@robertgshaw2): see if this can be moved out
297
+ if (cache_config.sliding_window is not None
298
+ and hasattr(config, "max_window_layers")):
299
+ raise ValueError("Sliding window for some but all layers is not "
300
+ "supported. This model uses sliding window "
301
+ "but `max_window_layers` = %s is less than "
302
+ "`num_hidden_layers` = %s. Please open an issue "
303
+ "to discuss this feature." % (
304
+ config.max_window_layers,
305
+ config.num_hidden_layers,
306
+ ))
307
+
308
super().__init__()
309
self.config = config
310
self.quant_config = quant_config
vllm/model_executor/models/starcoder2.py
-2
@@ -74,7 +74,6 @@ class Starcoder2Attention(nn.Module):
74
self.rope_theta = config.rope_theta
75
self.max_position_embeddings = config.max_position_embeddings
76
self.use_bias = config.use_bias
77
- self.sliding_window = config.sliding_window
77
78
self.qkv_proj = QKVParallelLinear(
79
self.hidden_size,
@@ -101,7 +100,6 @@ class Starcoder2Attention(nn.Module):
100
self.head_dim,
101
self.scaling,
102
num_kv_heads=self.num_kv_heads,
104
- sliding_window=self.sliding_window,
103
cache_config=cache_config,
104
quant_config=quant_config)
105
vllm/model_executor/models/xverse.py
-4
@@ -88,7 +88,6 @@ class XverseAttention(nn.Module):
88
max_position_embeddings: int = 8192,
89
quant_config: Optional[QuantizationConfig] = None,
90
bias: bool = False,
91
- sliding_window: Optional[int] = None,
91
cache_config: Optional[CacheConfig] = None,
92
) -> None:
93
super().__init__()
@@ -134,7 +133,6 @@ class XverseAttention(nn.Module):
133
self.head_dim,
134
self.scaling,
135
num_kv_heads=self.num_kv_heads,
137
- sliding_window=sliding_window,
136
cache_config=cache_config,
137
quant_config=quant_config)
138
@@ -167,7 +165,6 @@ class XverseDecoderLayer(nn.Module):
165
rope_scaling = getattr(config, "rope_scaling", None)
166
max_position_embeddings = getattr(config, "max_position_embeddings",
167
8192)
170
- sliding_window = getattr(config, "sliding_window", None)
168
self.self_attn = XverseAttention(
169
hidden_size=self.hidden_size,
170
num_heads=config.num_attention_heads,
@@ -178,7 +175,6 @@ class XverseDecoderLayer(nn.Module):
175
max_position_embeddings=max_position_embeddings,
176
quant_config=quant_config,
177
bias=getattr(config, "bias", False),
181
- sliding_window=sliding_window,
178
cache_config=cache_config,
179
)
180
self.mlp = XverseMLP(