A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[CI/Build][Doc] Update `gte-Qwen2-1.5B-instruct` usage (#18683)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
Cyrus Leung committed
May 26, 2025 at 11:27 UTC
fba06427043be95a6c3d4329aeba8b519c449f23
3 files changed
+2
-18
docs/models/supported_models.md
+1
-4
@@ -404,10 +404,7 @@ Specified using `--task embed`.
404
You should manually set mean pooling by passing `--override-pooler-config '{"pooling_type": "MEAN"}'`.
405
406
!!! note
407
- The HF implementation of `Alibaba-NLP/gte-Qwen2-1.5B-instruct` is hardcoded to use causal attention despite what is shown in `config.json`. To compare vLLM vs HF results,
408
- you should set `--hf-overrides '{"is_causal": true}'` in vLLM so that the two implementations are consistent with each other.
409
-
410
- For both the 1.5B and 7B variants, you also need to enable `--trust-remote-code` for the correct tokenizer to be loaded.
407
+ For `Alibaba-NLP/gte-Qwen2-*`, you need to enable `--trust-remote-code` for the correct tokenizer to be loaded.
408
See [relevant issue on HF Transformers](https://github.com/huggingface/transformers/issues/34882).
409
410
!!! note
tests/models/language/pooling/test_embedding.py
+1
-5
@@ -15,13 +15,12 @@ from ...utils import check_embeddings_close
15
marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
16
pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
17
pytest.param("intfloat/multilingual-e5-small"),
18
- pytest.param("Alibaba-NLP/gte-Qwen2-7B-instruct"),
18
+ pytest.param("Alibaba-NLP/gte-Qwen2-1.5B-instruct"),
19
# [Decoder-only]
20
pytest.param("BAAI/bge-multilingual-gemma2",
21
marks=[pytest.mark.core_model]),
22
pytest.param("intfloat/e5-mistral-7b-instruct",
23
marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
24
- pytest.param("Alibaba-NLP/gte-Qwen2-1.5B-instruct"),
24
pytest.param("ssmits/Qwen2-7B-Instruct-embed-base"),
25
# [Cross-Encoder]
26
pytest.param("sentence-transformers/stsb-roberta-base-v2"),
@@ -47,9 +46,6 @@ def test_models(
46
vllm_extra_kwargs["override_pooler_config"] = \
47
PoolerConfig(pooling_type="MEAN")
48
50
- if model == "Alibaba-NLP/gte-Qwen2-1.5B-instruct":
51
- vllm_extra_kwargs["hf_overrides"] = {"is_causal": True}
52
-
49
# The example_prompts has ending "\n", for example:
50
# "Write a short story about a robot that dreams for the first time.\n"
51
# sentence_transformers will strip the input texts, see:
tests/models/language/pooling/test_gte.py
-9
@@ -45,9 +45,6 @@ MODELS = [
45
EmbedModelInfo("Alibaba-NLP/gte-Qwen2-1.5B-instruct",
46
architecture="Qwen2ForCausalLM",
47
enable_test=True),
48
- EmbedModelInfo("Alibaba-NLP/gte-Qwen2-7B-instruct",
49
- architecture="Qwen2ForCausalLM",
50
- enable_test=False),
48
########## ModernBertModel
49
EmbedModelInfo("Alibaba-NLP/gte-modernbert-base",
50
architecture="ModernBertModel",
@@ -61,9 +58,6 @@ def test_models_mteb(hf_runner, vllm_runner,
58
from .mteb_utils import mteb_test_embed_models
59
60
vllm_extra_kwargs: dict[str, Any] = {}
64
- if model_info.name == "Alibaba-NLP/gte-Qwen2-1.5B-instruct":
65
- vllm_extra_kwargs["hf_overrides"] = {"is_causal": True}
66
-
61
if model_info.architecture == "GteNewModel":
62
vllm_extra_kwargs["hf_overrides"] = {"architectures": ["GteNewModel"]}
63
@@ -81,9 +75,6 @@ def test_models_correctness(hf_runner, vllm_runner, model_info: EmbedModelInfo,
75
example_prompts = [str(s).strip() for s in example_prompts]
76
77
vllm_extra_kwargs: dict[str, Any] = {}
84
- if model_info.name == "Alibaba-NLP/gte-Qwen2-1.5B-instruct":
85
- vllm_extra_kwargs["hf_overrides"] = {"is_causal": True}
86
-
78
if model_info.architecture == "GteNewModel":
79
vllm_extra_kwargs["hf_overrides"] = {"architectures": ["GteNewModel"]}
80