A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Guard `_may_reorder_batch` for encoder-only models on CPU (#24319) (#24348)
Signed-off-by: Remy <eunhwan.shin@dtonic.io> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
Remy committed
Sep 10, 2025 at 15:24 UTC
feaf202e93a18228cd34ae2e6698aece83a15e87
3 files changed
+26
-7
tests/models/language/pooling/test_embedding.py
+8
-2
@@ -27,11 +27,17 @@ from ...utils import check_embeddings_close
27
pytest.param("ssmits/Qwen2-7B-Instruct-embed-base",
28
marks=[pytest.mark.cpu_model]),
29
# [Encoder-only]
30
- pytest.param("BAAI/bge-base-en-v1.5", marks=[pytest.mark.core_model]),
30
+ pytest.param(
31
+ "BAAI/bge-base-en-v1.5",
32
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
33
+ ),
34
pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
35
pytest.param("intfloat/multilingual-e5-small"),
36
# [Cross-Encoder]
34
- pytest.param("sentence-transformers/stsb-roberta-base-v2"),
37
+ pytest.param(
38
+ "sentence-transformers/stsb-roberta-base-v2",
39
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
40
+ ),
41
],
42
)
43
def test_models(
vllm/config/__init__.py
+2
-1
@@ -3665,7 +3665,8 @@ class VllmConfig:
3665
# logger should only print warning message for hybrid models. As we
3666
# can't know whether the model is hybrid or not now, so we don't log
3667
# warning message here and will log it later.
3668
- if not (current_platform.is_cuda() or current_platform.is_rocm()):
3668
+ if not (current_platform.is_cuda() or current_platform.is_rocm()
3669
+ or current_platform.is_cpu()):
3670
# Hybrid KV cache manager is not supported on non-GPU platforms.
3671
self.scheduler_config.disable_hybrid_kv_cache_manager = True
3672
if self.kv_transfer_config is not None:
vllm/v1/worker/cpu_model_runner.py
+16
-4
@@ -55,11 +55,23 @@ class CPUModelRunner(GPUModelRunner):
55
raise ValueError("Multiple KVCacheGroups is not"
56
"currently supported with CPU model runner.")
57
58
- assert type(self.attn_groups[0]
59
- [0].metadata_builder) is TorchSDPAMetadataBuilderV1
58
+ # Guard against encoder-only / pooling models where `attn_groups`
59
+ # may be empty or lack the expected metadata_builder.
60
+ # Without this check, accessing `attn_groups[0][0]` would trigger
61
+ # an AssertionError on CPU backend.
62
+ if not hasattr(self, "attn_groups") or not self.attn_groups:
63
+ return
64
+ if not self.attn_groups[0]:
65
+ return
66
+
67
+ mb = getattr(self.attn_groups[0][0], "metadata_builder", None)
68
+ if not isinstance(mb, TorchSDPAMetadataBuilderV1):
69
+ # Encoder-only / rerank models do not benefit from reordering,
70
+ # so we safely skip here.
71
+ return
72
61
- self.attn_groups[0][0].metadata_builder.reorder_batch(
62
- self.input_batch, scheduler_output)
73
+ # Safe path for decoder/attention-heavy models
74
+ mb.reorder_batch(self.input_batch, scheduler_output)
75
76
def _postprocess_tensors(self) -> None:
77
# Note: replace device tensors with cpu tensors