A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model] Support Qwen2 embeddings and use tags to select model tests (#10184)
Cyrus Leung committed
Nov 15, 2024 at 12:23 UTC
b40cf6402e356a10415e969e648a32911fb9b8ec
19 files changed
+252
-178
.buildkite/run-cpu-test-ppc64le.sh
+3
-3
@@ -27,9 +27,9 @@ function cpu_tests() {
27
decord einops librosa peft Pillow sentence-transformers soundfile \
28
transformers_stream_generator matplotlib datamodel_code_generator
29
pip install torchvision --index-url https://download.pytorch.org/whl/cpu
30
- pytest -v -s tests/models/embedding/language
31
- pytest -v -s tests/models/encoder_decoder/language
32
- pytest -v -s tests/models/decoder_only/language/test_models.py
30
+ pytest -v -s tests/models/decoder_only/language -m cpu_model
31
+ pytest -v -s tests/models/embedding/language -m cpu_model
32
+ pytest -v -s tests/models/encoder_decoder/language -m cpu_model
33
pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
34
pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
35
.buildkite/run-cpu-test.sh
+3
-3
@@ -38,9 +38,9 @@ function cpu_tests() {
38
decord einops librosa peft Pillow sentence-transformers soundfile \
39
transformers_stream_generator matplotlib datamodel_code_generator
40
pip install torchvision --index-url https://download.pytorch.org/whl/cpu
41
- pytest -v -s tests/models/embedding/language
42
- pytest -v -s tests/models/encoder_decoder/language
43
- pytest -v -s tests/models/decoder_only/language/test_models.py
41
+ pytest -v -s tests/models/decoder_only/language -m cpu_model
42
+ pytest -v -s tests/models/embedding/language -m cpu_model
43
+ pytest -v -s tests/models/encoder_decoder/language -m cpu_model
44
pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
45
pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
46
.buildkite/test-pipeline.yaml
+23
-25
@@ -323,62 +323,60 @@ steps:
323
- pytest -v -s models/test_registry.py
324
- pytest -v -s models/test_initialization.py
325
326
-- label: Decoder-only Language Models Test (Standard) # 18min
326
+- label: Language Models Test (Standard) # 42min
327
#mirror_hardwares: [amd]
328
source_file_dependencies:
329
- vllm/
330
- tests/models/decoder_only/language
331
+ - tests/models/embedding/language
332
+ - tests/models/encoder_decoder/language
333
commands:
332
- - pytest -v -s models/decoder_only/language -m core_model
333
- - pytest -v -s models/decoder_only/language -m quant_model
334
+ - pytest -v -s models/decoder_only/language -m 'core_model or quant_model'
335
+ - pytest -v -s models/embedding/language -m core_model
336
+ - pytest -v -s models/embedding/vision_language -m core_model
337
335
-- label: Decoder-only Language Models Test (Extended) # 46min
338
+- label: Language Models Test (Extended) # 50min
339
nightly: true
340
source_file_dependencies:
341
- vllm/
342
- tests/models/decoder_only/language
343
+ - tests/models/embedding/language
344
+ - tests/models/encoder_decoder/language
345
commands:
346
- pytest -v -s models/decoder_only/language -m 'not core_model and not quant_model'
347
+ - pytest -v -s models/embedding/language -m 'not core_model'
348
+ - pytest -v -s models/embedding/vision_language -m 'not core_model'
349
343
-- label: Decoder-only Multi-Modal Models Test (Standard) # 22min
350
+- label: Multi-Modal Models Test (Standard) # 26min
351
#mirror_hardwares: [amd]
352
source_file_dependencies:
353
- vllm/
354
- tests/models/decoder_only/audio_language
355
- tests/models/decoder_only/vision_language
356
+ - tests/models/embedding/vision_language
357
+ - tests/models/encoder_decoder/vision_language
358
commands:
350
- - pytest -v -s models/decoder_only/audio_language -m core_model
351
- - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m core_model
352
- # No tests under this group for now
353
- # - pytest -v -s models/decoder_only/audio_language -m quant_model
354
- - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m quant_model
359
+ - pytest -v -s models/decoder_only/audio_language -m 'core_model or quant_model'
360
+ - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'core_model or quant_model'
361
+ - pytest -v -s models/encoder_decoder/language -m core_model
362
+ - pytest -v -s models/encoder_decoder/vision_language -m core_model
363
356
-- label: Decoder-only Multi-Modal Models Test (Extended) # 1h10m
364
+- label: Multi-Modal Models Test (Extended) # 1h15m
365
nightly: true
366
source_file_dependencies:
367
- vllm/
368
- tests/models/decoder_only/audio_language
369
- tests/models/decoder_only/vision_language
370
+ - tests/models/embedding/vision_language
371
+ - tests/models/encoder_decoder/vision_language
372
commands:
373
- pytest -v -s models/decoder_only/audio_language -m 'not core_model and not quant_model'
374
# HACK - run phi3v tests separately to sidestep this transformers bug
375
# https://github.com/huggingface/transformers/issues/34307
376
- pytest -v -s models/decoder_only/vision_language/test_phi3v.py
377
- pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
368
-
369
-- label: Other Models Test # 20min
370
- #mirror_hardwares: [amd]
371
- source_file_dependencies:
372
- - vllm/
373
- - tests/models/embedding/language
374
- - tests/models/embedding/vision_language
375
- - tests/models/encoder_decoder/language
376
- - tests/models/encoder_decoder/vision_language
377
- commands:
378
- - pytest -v -s models/embedding/language
379
- - pytest -v -s models/embedding/vision_language
380
- - pytest -v -s models/encoder_decoder/language
381
- - pytest -v -s models/encoder_decoder/vision_language
378
+ - pytest -v -s models/encoder_decoder/language -m 'not core_model'
379
+ - pytest -v -s models/encoder_decoder/vision_language -m 'not core_model'
380
381
# This test is used only in PR development phase to test individual models and should never run on main
382
- label: Custom Models Test
docs/source/models/supported_models.rst
+9
-4
@@ -330,11 +330,16 @@ Text Embedding
330
- :code:`BAAI/bge-multilingual-gemma2`, etc.
331
-
332
- ✅︎
333
- * - :code:`MistralModel`
334
- - Mistral-based
333
+ * - :code:`LlamaModel`, :code:`LlamaForCausalLM`, :code:`MistralModel`, etc.
334
+ - Llama-based
335
- :code:`intfloat/e5-mistral-7b-instruct`, etc.
336
- ✅︎
337
- ✅︎
338
+ * - :code:`Qwen2Model`, :code:`Qwen2ForCausalLM`
339
+ - Qwen2-based
340
+ - :code:`ssmits/Qwen2-7B-Instruct-embed-base`, :code:`Alibaba-NLP/gte-Qwen2-1.5B-instruct`, etc.
341
+ - ✅︎
342
+ - ✅︎
343
344
.. important::
345
Some model architectures support both generation and embedding tasks.
@@ -355,7 +360,7 @@ Reward Modeling
360
* - :code:`Qwen2ForRewardModel`
361
- Qwen2-based
362
- :code:`Qwen/Qwen2.5-Math-RM-72B`, etc.
358
- -
363
+ - ✅︎
364
- ✅︎
365
366
.. note::
@@ -376,7 +381,7 @@ Classification
381
* - :code:`Qwen2ForSequenceClassification`
382
- Qwen2-based
383
- :code:`jason9693/Qwen2.5-1.5B-apeach`, etc.
379
- -
384
+ - ✅︎
385
- ✅︎
386
387
.. note::
tests/models/decoder_only/language/test_jamba.py
+4
-14
@@ -33,6 +33,10 @@ def test_models(
33
34
with vllm_runner(model, dtype=dtype) as vllm_model:
35
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
36
+ # This test is for verifying whether the model's extra_repr
37
+ # can be printed correctly.
38
+ print(vllm_model.model.llm_engine.model_executor.driver_worker.
39
+ model_runner.model)
40
41
for i in range(len(example_prompts)):
42
hf_output_ids, hf_output_str = hf_outputs[i]
@@ -293,17 +297,3 @@ def test_jamba_distributed_produces_identical_generation(
297
name_0="vllm_tp_1",
298
name_1="vllm_tp_2",
299
)
296
-
297
-
298
-@pytest.mark.parametrize("model", MODELS)
299
-@pytest.mark.parametrize("dtype", ["float"])
300
-def test_model_print(
301
- vllm_runner,
302
- model: str,
303
- dtype: str,
304
-) -> None:
305
- with vllm_runner(model, dtype=dtype) as vllm_model:
306
- # This test is for verifying whether the model's extra_repr
307
- # can be printed correctly.
308
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
309
- model_runner.model)
tests/models/decoder_only/language/test_mamba.py
+4
-14
@@ -51,6 +51,10 @@ def test_models(
51
52
with vllm_runner(model, dtype=dtype) as vllm_model:
53
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54
+ # This test is for verifying whether the model's extra_repr
55
+ # can be printed correctly.
56
+ print(vllm_model.model.llm_engine.model_executor.driver_worker.
57
+ model_runner.model)
58
59
for i in range(len(example_prompts)):
60
hf_output_ids, hf_output_str = hf_outputs[i]
@@ -279,17 +283,3 @@ def test_state_cleanup(
283
except ValueError:
284
pytest.fail("Mamba inner state wasn't cleaned up between states, "
285
"could be related to finished_requests_ids")
282
-
283
-
284
-@pytest.mark.parametrize("model", MODELS)
285
-@pytest.mark.parametrize("dtype", ["float"])
286
-def test_model_print(
287
- vllm_runner,
288
- model: str,
289
- dtype: str,
290
-) -> None:
291
- with vllm_runner(model, dtype=dtype) as vllm_model:
292
- # This test is for verifying whether the model's extra_repr
293
- # can be printed correctly.
294
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
295
- model_runner.model)
tests/models/decoder_only/language/test_models.py
+43
-28
@@ -4,37 +4,52 @@ Run `pytest tests/models/test_models.py`.
4
"""
5
import pytest
6
7
-from vllm.platforms import current_platform
8
-
7
from ...utils import check_logprobs_close
8
11
-MODELS = [
12
- "facebook/opt-125m", # opt
13
- "openai-community/gpt2", # gpt2
14
- # "Milos/slovak-gpt-j-405M", # gptj
15
- # "bigcode/tiny_starcoder_py", # gpt_bigcode
16
- # "EleutherAI/pythia-70m", # gpt_neox
17
- "bigscience/bloom-560m", # bloom - testing alibi slopes
18
- "microsoft/phi-2", # phi
19
- # "stabilityai/stablelm-3b-4e1t", # stablelm
20
- # "bigcode/starcoder2-3b", # starcoder2
21
- "google/gemma-1.1-2b-it", # gemma
22
- "Qwen/Qwen2.5-0.5B-Instruct", # qwen2
23
- "meta-llama/Llama-3.2-1B-Instruct", # llama
24
-]
25
-
26
-if not current_platform.is_cpu():
27
- MODELS += [
28
- # fused_moe which not supported on CPU
29
- "openbmb/MiniCPM3-4B",
30
- ]
31
-
32
-target_dtype = "half"
33
-
9
35
-@pytest.mark.core_model
36
-@pytest.mark.parametrize("model", MODELS)
37
-@pytest.mark.parametrize("dtype", [target_dtype])
10
+@pytest.mark.parametrize(
11
+ "model",
12
+ [
13
+ pytest.param(
14
+ "bigscience/bloom-560m", # bloom - testing alibi slopes
15
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
16
+ ),
17
+ pytest.param(
18
+ "openai-community/gpt2", # gpt2
19
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
20
+ ),
21
+ pytest.param("Milos/slovak-gpt-j-405M"), # gptj
22
+ pytest.param("bigcode/tiny_starcoder_py"), # gpt_bigcode
23
+ pytest.param("EleutherAI/pythia-70m"), # gpt_neox
24
+ pytest.param(
25
+ "google/gemma-1.1-2b-it", # gemma
26
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
27
+ ),
28
+ pytest.param(
29
+ "meta-llama/Llama-3.2-1B-Instruct", # llama
30
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
31
+ ),
32
+ pytest.param(
33
+ "openbmb/MiniCPM3-4B",
34
+ # fused_moe not supported on CPU
35
+ marks=[pytest.mark.core_model],
36
+ ),
37
+ pytest.param(
38
+ "facebook/opt-125m", # opt
39
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
40
+ ),
41
+ pytest.param(
42
+ "microsoft/phi-2", # phi
43
+ marks=[pytest.mark.core_model],
44
+ ),
45
+ pytest.param(
46
+ "Qwen/Qwen2.5-0.5B-Instruct", # qwen2
47
+ marks=[pytest.mark.core_model],
48
+ ),
49
+ pytest.param("stabilityai/stablelm-3b-4e1t"), # stablelm
50
+ pytest.param("bigcode/starcoder2-3b"), # starcoder2
51
+ ])
52
+@pytest.mark.parametrize("dtype", ["half"])
53
@pytest.mark.parametrize("max_tokens", [32])
54
@pytest.mark.parametrize("num_logprobs", [5])
55
def test_models(
tests/models/embedding/language/test_cls_models.py
+11
-19
@@ -9,10 +9,14 @@ import pytest
9
import torch
10
from transformers import AutoModelForSequenceClassification
11
12
-CLASSIFICATION_MODELS = ["jason9693/Qwen2.5-1.5B-apeach"]
12
14
-
15
-@pytest.mark.parametrize("model", CLASSIFICATION_MODELS)
13
+@pytest.mark.parametrize(
14
+ "model",
15
+ [
16
+ pytest.param("jason9693/Qwen2.5-1.5B-apeach",
17
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
18
+ ],
19
+)
20
@pytest.mark.parametrize("dtype", ["float"])
21
def test_classification_models(
22
hf_runner,
@@ -23,31 +27,19 @@ def test_classification_models(
27
) -> None:
28
with vllm_runner(model, dtype=dtype) as vllm_model:
29
vllm_outputs = vllm_model.classify(example_prompts)
30
+ # This test is for verifying whether the model's extra_repr
31
+ # can be printed correctly.
32
+ print(vllm_model.model.llm_engine.model_executor.driver_worker.
33
+ model_runner.model)
34
35
with hf_runner(model,
36
dtype=dtype,
37
auto_cls=AutoModelForSequenceClassification) as hf_model:
38
hf_outputs = hf_model.classify(example_prompts)
39
32
- print(hf_outputs, vllm_outputs)
33
-
40
# check logits difference
41
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
42
hf_output = torch.tensor(hf_output)
43
vllm_output = torch.tensor(vllm_output)
44
45
assert torch.allclose(hf_output, vllm_output, 1e-3)
40
-
41
-
42
-@pytest.mark.parametrize("model", CLASSIFICATION_MODELS)
43
-@pytest.mark.parametrize("dtype", ["float"])
44
-def test_classification_model_print(
45
- vllm_runner,
46
- model: str,
47
- dtype: str,
48
-) -> None:
49
- with vllm_runner(model, dtype=dtype) as vllm_model:
50
- # This test is for verifying whether the model's extra_repr
51
- # can be printed correctly.
52
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
53
- model_runner.model)
tests/models/embedding/language/test_embedding.py
+22
-20
@@ -4,25 +4,25 @@ Run `pytest tests/models/embedding/language/test_embedding.py`.
4
"""
5
import pytest
6
7
-from vllm.utils import current_platform
8
-
7
from ..utils import check_embeddings_close
8
11
-# Model, Guard
12
-MODELS = [
13
- "intfloat/e5-mistral-7b-instruct",
14
- "BAAI/bge-base-en-v1.5",
15
- "BAAI/bge-multilingual-gemma2",
16
- "intfloat/multilingual-e5-large",
17
-]
18
-
19
-ENCODER_ONLY = [
20
- "BAAI/bge-base-en-v1.5",
21
- "intfloat/multilingual-e5-large",
22
-]
9
24
-
25
-@pytest.mark.parametrize("model", MODELS)
10
+@pytest.mark.parametrize(
11
+ "model",
12
+ [
13
+ # [Encoder-only]
14
+ pytest.param("BAAI/bge-base-en-v1.5",
15
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
16
+ pytest.param("intfloat/multilingual-e5-large"),
17
+ # [Encoder-decoder]
18
+ pytest.param("intfloat/e5-mistral-7b-instruct",
19
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
20
+ pytest.param("BAAI/bge-multilingual-gemma2",
21
+ marks=[pytest.mark.core_model]),
22
+ pytest.param("ssmits/Qwen2-7B-Instruct-embed-base"),
23
+ pytest.param("Alibaba-NLP/gte-Qwen2-1.5B-instruct"),
24
+ ],
25
+)
26
@pytest.mark.parametrize("dtype", ["half"])
27
def test_models(
28
hf_runner,
@@ -31,9 +31,6 @@ def test_models(
31
model,
32
dtype: str,
33
) -> None:
34
- if model not in ENCODER_ONLY and current_platform.is_cpu():
35
- pytest.skip("Skip large embedding models test on CPU.")
36
-
34
# The example_prompts has ending "\n", for example:
35
# "Write a short story about a robot that dreams for the first time.\n"
36
# sentence_transformers will strip the input texts, see:
@@ -46,8 +43,13 @@ def test_models(
43
is_sentence_transformer=True) as hf_model:
44
hf_outputs = hf_model.encode(example_prompts)
45
49
- with vllm_runner(model, dtype=dtype, max_model_len=None) as vllm_model:
46
+ with vllm_runner(model, task="embedding", dtype=dtype,
47
+ max_model_len=None) as vllm_model:
48
vllm_outputs = vllm_model.encode(example_prompts)
49
+ # This test is for verifying whether the model's extra_repr
50
+ # can be printed correctly.
51
+ print(vllm_model.model.llm_engine.model_executor.driver_worker.
52
+ model_runner.model)
53
54
check_embeddings_close(
55
embeddings_0_lst=hf_outputs,
tests/models/embedding/vision_language/test_llava_next.py
+2
@@ -88,6 +88,7 @@ def _run_test(
88
89
@pytest.mark.skipif(transformers.__version__.startswith("4.46"),
90
reason="Model broken with changes in transformers 4.46")
91
+@pytest.mark.core_model
92
@pytest.mark.parametrize("model", MODELS)
93
@pytest.mark.parametrize("dtype", ["half"])
94
def test_models_text(
@@ -112,6 +113,7 @@ def test_models_text(
113
114
115
@large_gpu_test(min_gb=48)
116
+@pytest.mark.core_model
117
@pytest.mark.parametrize("model", MODELS)
118
@pytest.mark.parametrize("dtype", ["half"])
119
def test_models_image(
tests/models/embedding/vision_language/test_phi3v.py
+2
@@ -74,6 +74,7 @@ def _run_test(
74
)
75
76
77
+@pytest.mark.core_model
78
@pytest.mark.parametrize("model", MODELS)
79
@pytest.mark.parametrize("dtype", ["half"])
80
def test_models_text(
@@ -98,6 +99,7 @@ def test_models_text(
99
100
101
@large_gpu_test(min_gb=48)
102
+@pytest.mark.core_model
103
@pytest.mark.parametrize("model", MODELS)
104
@pytest.mark.parametrize("dtype", ["half"])
105
def test_models_image(
tests/models/encoder_decoder/language/test_bart.py
+8
-3
@@ -14,8 +14,6 @@ from ....conftest import (DecoderPromptType, ExplicitEncoderDecoderPrompt,
14
from ....utils import multi_gpu_test
15
from ...utils import check_logprobs_close
16
17
-MODELS = ["facebook/bart-base", "facebook/bart-large-cnn"]
18
-
17
18
def vllm_to_hf_output(
19
vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
@@ -170,7 +168,14 @@ def run_test(
168
)
169
170
173
-@pytest.mark.parametrize("model", MODELS)
171
+@pytest.mark.parametrize(
172
+ "model",
173
+ [
174
+ pytest.param("facebook/bart-base",
175
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
176
+ pytest.param("facebook/bart-large-cnn"),
177
+ ],
178
+)
179
@pytest.mark.parametrize("dtype", ["float", "bfloat16"])
180
@pytest.mark.parametrize("max_tokens", [64])
181
@pytest.mark.parametrize("num_logprobs", [5])
tests/models/encoder_decoder/vision_language/test_mllama.py
+3
@@ -233,6 +233,7 @@ def clear_cache():
233
234
235
@large_gpu_test(min_gb=48)
236
+@pytest.mark.core_model
237
@pytest.mark.parametrize("model", models)
238
@pytest.mark.parametrize(
239
"sizes",
@@ -278,6 +279,7 @@ def test_models_single_leading_image(hf_runner, vllm_runner, image_assets,
279
280
281
@large_gpu_test(min_gb=48)
282
+@pytest.mark.core_model
283
@pytest.mark.parametrize("model", models)
284
@pytest.mark.parametrize("dtype", ["bfloat16"])
285
@pytest.mark.parametrize("max_tokens", [128])
@@ -326,6 +328,7 @@ def test_models_multi_leading_images(hf_runner, vllm_runner, image_assets,
328
329
330
@large_gpu_test(min_gb=48)
331
+@pytest.mark.core_model
332
@pytest.mark.parametrize("model", models)
333
@pytest.mark.parametrize("dtype", ["bfloat16"])
334
@pytest.mark.parametrize("max_tokens", [128])
tests/models/registry.py
+4
@@ -129,9 +129,13 @@ _EMBEDDING_EXAMPLE_MODELS = {
129
# [Text-only]
130
"BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
131
"Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
132
+ "LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
133
"MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
134
+ "Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
135
"Qwen2ForRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-RM-72B"),
136
"Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"), # noqa: E501
137
+ "RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2"), # noqa: E501
138
+ "XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-large"),
139
# [Multimodal]
140
"LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"),
141
"Phi3VForCausalLM": _HfExamplesInfo("TIGER-Lab/VLM2Vec-Full",
tests/models/test_registry.py
+2
-2
@@ -77,8 +77,8 @@ def test_registry_is_pp(model_arch, is_pp, init_cuda):
77
78
79
def test_hf_registry_coverage():
80
- untested_archs = (HF_EXAMPLE_MODELS.get_supported_archs() -
81
- set(ModelRegistry.get_supported_archs()))
80
+ untested_archs = (ModelRegistry.get_supported_archs() -
81
+ HF_EXAMPLE_MODELS.get_supported_archs())
82
83
assert not untested_archs, (
84
"Please add the following architectures to "
vllm/model_executor/models/qwen2.py
+99
-13
@@ -37,6 +37,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37
QKVParallelLinear,
38
RowParallelLinear)
39
from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
+from vllm.model_executor.layers.pooler import Pooler, PoolingType
41
from vllm.model_executor.layers.quantization import QuantizationConfig
42
from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -44,8 +45,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
45
ParallelLMHead, VocabParallelEmbedding)
46
from vllm.model_executor.model_loader.weight_utils import (
47
default_weight_loader, maybe_remap_kv_scale_name)
48
+from vllm.model_executor.pooling_metadata import PoolingMetadata
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
48
-from vllm.sequence import IntermediateTensors
50
+from vllm.sequence import IntermediateTensors, PoolerOutput
51
52
from .interfaces import SupportsLoRA, SupportsPP
53
from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
@@ -247,6 +249,18 @@ class Qwen2Model(nn.Module):
249
cache_config = vllm_config.cache_config
250
quant_config = vllm_config.quant_config
251
252
+ # TODO (@robertgshaw2): see if this can be moved out
253
+ if (cache_config.sliding_window is not None
254
+ and hasattr(config, "max_window_layers")):
255
+ raise ValueError("Sliding window for some but all layers is not "
256
+ "supported. This model uses sliding window "
257
+ "but `max_window_layers` = {} is less than "
258
+ "`num_hidden_layers` = {}. Please open an issue "
259
+ "to discuss this feature.".format(
260
+ config.max_window_layers,
261
+ config.num_hidden_layers,
262
+ ))
263
+
264
self.config = config
265
self.padding_idx = config.pad_token_id
266
self.vocab_size = config.vocab_size
@@ -405,20 +419,9 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
419
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
420
super().__init__()
421
config = vllm_config.model_config.hf_config
408
- cache_config = vllm_config.cache_config
422
quant_config = vllm_config.quant_config
423
lora_config = vllm_config.lora_config
411
- # TODO (@robertgshaw2): see if this can be moved out
412
- if (cache_config.sliding_window is not None
413
- and hasattr(config, "max_window_layers")):
414
- raise ValueError("Sliding window for some but all layers is not "
415
- "supported. This model uses sliding window "
416
- "but `max_window_layers` = {} is less than "
417
- "`num_hidden_layers` = {}. Please open an issue "
418
- "to discuss this feature.".format(
419
- config.max_window_layers,
420
- config.num_hidden_layers,
421
- ))
424
+ pooler_config = vllm_config.model_config.pooler_config
425
426
self.config = config
427
self.lora_config = lora_config
@@ -438,6 +441,15 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
441
442
self.logits_processor = LogitsProcessor(config.vocab_size)
443
self.sampler = get_sampler()
444
+
445
+ # The same model class supports both language generation and embedding
446
+ # because the architecture name is the same
447
+ self._pooler = Pooler.from_config_with_defaults(
448
+ pooler_config,
449
+ pooling_type=PoolingType.LAST,
450
+ normalize=True,
451
+ softmax=False)
452
+
453
self.make_empty_intermediate_tensors = (
454
self.model.make_empty_intermediate_tensors)
455
@@ -475,6 +487,13 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
487
next_tokens = self.sampler(logits, sampling_metadata)
488
return next_tokens
489
490
+ def pooler(
491
+ self,
492
+ hidden_states: torch.Tensor,
493
+ pooling_metadata: PoolingMetadata,
494
+ ) -> Optional[PoolerOutput]:
495
+ return self._pooler(hidden_states, pooling_metadata)
496
+
497
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
498
loader = AutoWeightsLoader(
499
self,
@@ -482,3 +501,70 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
501
if self.config.tie_word_embeddings else None),
502
)
503
loader.load_weights(weights)
504
+
505
+
506
+class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
507
+ packed_modules_mapping = {
508
+ "qkv_proj": [
509
+ "q_proj",
510
+ "k_proj",
511
+ "v_proj",
512
+ ],
513
+ "gate_up_proj": [
514
+ "gate_proj",
515
+ "up_proj",
516
+ ],
517
+ }
518
+
519
+ # LoRA specific attributes
520
+ supported_lora_modules = [
521
+ "qkv_proj",
522
+ "o_proj",
523
+ "gate_up_proj",
524
+ "down_proj",
525
+ ]
526
+ embedding_modules = {}
527
+ embedding_padding_modules = []
528
+
529
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
530
+ super().__init__()
531
+ config = vllm_config.model_config.hf_config
532
+ quant_config = vllm_config.quant_config
533
+ lora_config = vllm_config.lora_config
534
+ pooler_config = vllm_config.model_config.pooler_config
535
+
536
+ self.config = config
537
+ self.lora_config = lora_config
538
+
539
+ self.quant_config = quant_config
540
+ self.model = Qwen2Model(vllm_config=vllm_config,
541
+ prefix=maybe_prefix(prefix, "model"))
542
+
543
+ self._pooler = Pooler.from_config_with_defaults(
544
+ pooler_config,
545
+ pooling_type=PoolingType.MEAN,
546
+ normalize=True,
547
+ softmax=False)
548
+
549
+ def forward(
550
+ self,
551
+ input_ids: torch.Tensor,
552
+ positions: torch.Tensor,
553
+ kv_caches: List[torch.Tensor],
554
+ attn_metadata: AttentionMetadata,
555
+ intermediate_tensors: Optional[IntermediateTensors] = None,
556
+ ) -> torch.Tensor:
557
+ return self.model(input_ids, positions, kv_caches, attn_metadata,
558
+ intermediate_tensors)
559
+
560
+ def pooler(
561
+ self,
562
+ hidden_states: torch.Tensor,
563
+ pooling_metadata: PoolingMetadata,
564
+ ) -> Optional[PoolerOutput]:
565
+ return self._pooler(hidden_states, pooling_metadata)
566
+
567
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
568
+ loader = AutoWeightsLoader(self,
569
+ ignore_unexpected_prefixes=["lm_head."])
570
+ loader.load_weights(weights)
vllm/model_executor/models/qwen2_cls.py
+2
-13
@@ -17,10 +17,11 @@ from vllm.model_executor.models.qwen2 import Qwen2Model
17
from vllm.model_executor.pooling_metadata import PoolingMetadata
18
from vllm.sequence import IntermediateTensors, PoolerOutput
19
20
+from .interfaces import SupportsLoRA, SupportsPP
21
from .utils import AutoWeightsLoader, maybe_prefix
22
23
23
-class Qwen2ForSequenceClassification(nn.Module):
24
+class Qwen2ForSequenceClassification(nn.Module, SupportsLoRA, SupportsPP):
25
packed_modules_mapping = {
26
"qkv_proj": [
27
"q_proj",
@@ -46,21 +47,9 @@ class Qwen2ForSequenceClassification(nn.Module):
47
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
48
super().__init__()
49
config = vllm_config.model_config.hf_config
49
- cache_config = vllm_config.cache_config
50
quant_config = vllm_config.quant_config
51
lora_config = vllm_config.lora_config
52
pooler_config = vllm_config.model_config.pooler_config
53
- # TODO (@robertgshaw2): see if this can be moved out
54
- if (cache_config.sliding_window is not None
55
- and hasattr(config, "max_window_layers")):
56
- raise ValueError("Sliding window for some but all layers is not "
57
- "supported. This model uses sliding window "
58
- "but `max_window_layers` = {} is less than "
59
- "`num_hidden_layers` = {}. Please open an issue "
60
- "to discuss this feature.".format(
61
- config.max_window_layers,
62
- config.num_hidden_layers,
63
- ))
53
54
self.config = config
55
self.lora_config = lora_config
vllm/model_executor/models/qwen2_rm.py
+2
-14
@@ -16,7 +16,7 @@ from vllm.model_executor.layers.pooler import Pooler, PoolingType
16
from vllm.model_executor.pooling_metadata import PoolingMetadata
17
from vllm.sequence import IntermediateTensors, PoolerOutput
18
19
-from .interfaces import SupportsPP
19
+from .interfaces import SupportsLoRA, SupportsPP
20
from .qwen2 import Qwen2Model
21
from .utils import AutoWeightsLoader, maybe_prefix
22
@@ -32,7 +32,7 @@ class ReLU(nn.Module):
32
return self.activation(input)
33
34
35
-class Qwen2ForRewardModel(nn.Module, SupportsPP):
35
+class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
36
packed_modules_mapping = {
37
"qkv_proj": [
38
"q_proj",
@@ -58,21 +58,9 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
58
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
59
super().__init__()
60
config = vllm_config.model_config.hf_config
61
- cache_config = vllm_config.cache_config
61
quant_config = vllm_config.quant_config
62
lora_config = vllm_config.lora_config
63
pooler_config = vllm_config.model_config.pooler_config
65
- # TODO (@robertgshaw2): see if this can be moved out
66
- if (cache_config.sliding_window is not None
67
- and hasattr(config, "max_window_layers")):
68
- raise ValueError("Sliding window for some but all layers is not "
69
- "supported. This model uses sliding window "
70
- "but `max_window_layers` = {} is less than "
71
- "`num_hidden_layers` = {}. Please open an issue "
72
- "to discuss this feature.".format(
73
- config.max_window_layers,
74
- config.num_hidden_layers,
75
- ))
64
65
self.config = config
66
self.lora_config = lora_config
vllm/model_executor/models/registry.py
+6
-3
@@ -11,7 +11,8 @@ import tempfile
11
from abc import ABC, abstractmethod
12
from dataclasses import dataclass, field
13
from functools import lru_cache
14
-from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union
14
+from typing import (AbstractSet, Callable, Dict, List, Optional, Tuple, Type,
15
+ TypeVar, Union)
16
17
import cloudpickle
18
import torch.nn as nn
@@ -110,6 +111,8 @@ _EMBEDDING_MODELS = {
111
},
112
"MistralModel": ("llama", "LlamaEmbeddingModel"),
113
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
114
+ "Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
115
+ "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
116
"Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
117
"Qwen2ForSequenceClassification": ("qwen2_cls", "Qwen2ForSequenceClassification"), # noqa: E501
118
# [Multimodal]
@@ -301,8 +304,8 @@ class _ModelRegistry:
304
# Keyed by model_arch
305
models: Dict[str, _BaseRegisteredModel] = field(default_factory=dict)
306
304
- def get_supported_archs(self) -> List[str]:
305
- return list(self.models.keys())
307
+ def get_supported_archs(self) -> AbstractSet[str]:
308
+ return self.models.keys()
309
310
def register_model(
311
self,