A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix][CI/Build] Fix failing Mteb CI (#26638)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Oct 12, 2025 at 17:42 UTC
045b396d090f4a16fbba760bef86e9a24a7ba9ce
5 files changed
+13
-2
tests/models/language/pooling_mteb_test/mteb_utils.py
+1
-1
@@ -191,7 +191,7 @@ def mteb_test_embed_models(
191
with vllm_runner(
192
model_info.name,
193
runner="pooling",
194
- max_model_len=None,
194
+ max_model_len=model_info.max_model_len,
195
**vllm_extra_kwargs,
196
) as vllm_model:
197
model_config = vllm_model.llm.llm_engine.model_config
tests/models/language/pooling_mteb_test/test_jina.py
+5
@@ -25,6 +25,11 @@ EMBEDDING_MODELS = [
25
mteb_score=0.824413164,
26
architecture="XLMRobertaModel",
27
is_matryoshka=True,
28
+ # The default max length of the model is 8194, which will crash
29
+ # CUDAGraph due to odd length for Gemm. We set it to 8192 to avoid
30
+ # avoid this issue.
31
+ max_model_len=8192,
32
+ dtype="float32",
33
)
34
]
35
tests/models/language/pooling_mteb_test/test_st_projector.py
+1
@@ -23,6 +23,7 @@ ST_PROJECTOR_MODELS = [
23
architecture="Gemma3TextModel",
24
mteb_score=0.7473819294684156,
25
enable_test=True,
26
+ dtype="float32",
27
),
28
]
29
tests/models/utils.py
+1
@@ -369,6 +369,7 @@ class ModelInfo:
369
name: str
370
architecture: str = ""
371
dtype: str = "auto"
372
+ max_model_len: Optional[int] = None
373
hf_dtype: str = "float32"
374
hf_overrides: Optional[dict[str, Any]] = None
375
default_pooling_type: str = ""
vllm/model_executor/layers/layernorm.py
+5
-1
@@ -318,7 +318,11 @@ class GemmaRMSNorm(CustomOp):
318
"""PyTorch-native implementation equivalent to forward()."""
319
orig_dtype = x.dtype
320
if residual is not None:
321
- x = x + residual.float() if orig_dtype == torch.float16 else x + residual
321
+ x = (
322
+ x.float() + residual.float()
323
+ if orig_dtype == torch.float16
324
+ else x + residual
325
+ )
326
residual = x
327
328
x = x.float()