A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
skip fusedmoe layer for start_load_kv (#21378)
Signed-off-by: calvin chen <wen.chen@dynamia.ai>
Calvin Chen committed
Jul 29, 2025 at 09:59 UTC
e18f0851033fbc4ef55c1989411f2a5666b518c6
1 file changed
+10
-2
vllm/distributed/kv_transfer/kv_connector/v1/p2p/p2p_nccl_connector.py
+10
-2
@@ -192,8 +192,16 @@ class P2pNcclConnector(KVConnectorBase_V1):
192
# Load the KV for each request each layer
193
for request in metadata.requests:
194
for layer_name in forward_context.no_compile_layers:
195
- attn_layer = forward_context.no_compile_layers[layer_name]
196
- kv_cache_layer = attn_layer.kv_cache[ \
195
+ layer = forward_context.no_compile_layers[layer_name]
196
+
197
+ # Only process layers that have kv_cache
198
+ # attribute (attention layers) Skip non-attention
199
+ # layers like FusedMoE
200
+ kv_cache = getattr(layer, 'kv_cache', None)
201
+ if kv_cache is None:
202
+ continue
203
+
204
+ kv_cache_layer = kv_cache[ \
205
forward_context.virtual_engine]
206
207
kv_cache = self.p2p_nccl_engine.recv_tensor(