@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b40cf6402e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Model] Support Qwen2 embeddings and use tags to select model tests (#10184)

Cyrus Leung committed Nov 15, 2024 at 12:23 UTC b40cf6402e356a10415e969e648a32911fb9b8ec
19 files changed +252 -178
.buildkite/run-cpu-test-ppc64le.sh
+3 -3
@@ -27,9 +27,9 @@ function cpu_tests() {
27 decord einops librosa peft Pillow sentence-transformers soundfile \
28 transformers_stream_generator matplotlib datamodel_code_generator
29 pip install torchvision --index-url https://download.pytorch.org/whl/cpu
30 - pytest -v -s tests/models/embedding/language
31 - pytest -v -s tests/models/encoder_decoder/language
32 - pytest -v -s tests/models/decoder_only/language/test_models.py
30 + pytest -v -s tests/models/decoder_only/language -m cpu_model
31 + pytest -v -s tests/models/embedding/language -m cpu_model
32 + pytest -v -s tests/models/encoder_decoder/language -m cpu_model
33 pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
34 pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
35
.buildkite/run-cpu-test.sh
+3 -3
@@ -38,9 +38,9 @@ function cpu_tests() {
38 decord einops librosa peft Pillow sentence-transformers soundfile \
39 transformers_stream_generator matplotlib datamodel_code_generator
40 pip install torchvision --index-url https://download.pytorch.org/whl/cpu
41 - pytest -v -s tests/models/embedding/language
42 - pytest -v -s tests/models/encoder_decoder/language
43 - pytest -v -s tests/models/decoder_only/language/test_models.py
41 + pytest -v -s tests/models/decoder_only/language -m cpu_model
42 + pytest -v -s tests/models/embedding/language -m cpu_model
43 + pytest -v -s tests/models/encoder_decoder/language -m cpu_model
44 pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
45 pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
46
.buildkite/test-pipeline.yaml
+23 -25
@@ -323,62 +323,60 @@ steps:
323 - pytest -v -s models/test_registry.py
324 - pytest -v -s models/test_initialization.py
325
326 -- label: Decoder-only Language Models Test (Standard) # 18min
326 +- label: Language Models Test (Standard) # 42min
327 #mirror_hardwares: [amd]
328 source_file_dependencies:
329 - vllm/
330 - tests/models/decoder_only/language
331 + - tests/models/embedding/language
332 + - tests/models/encoder_decoder/language
333 commands:
332 - - pytest -v -s models/decoder_only/language -m core_model
333 - - pytest -v -s models/decoder_only/language -m quant_model
334 + - pytest -v -s models/decoder_only/language -m 'core_model or quant_model'
335 + - pytest -v -s models/embedding/language -m core_model
336 + - pytest -v -s models/embedding/vision_language -m core_model
337
335 -- label: Decoder-only Language Models Test (Extended) # 46min
338 +- label: Language Models Test (Extended) # 50min
339 nightly: true
340 source_file_dependencies:
341 - vllm/
342 - tests/models/decoder_only/language
343 + - tests/models/embedding/language
344 + - tests/models/encoder_decoder/language
345 commands:
346 - pytest -v -s models/decoder_only/language -m 'not core_model and not quant_model'
347 + - pytest -v -s models/embedding/language -m 'not core_model'
348 + - pytest -v -s models/embedding/vision_language -m 'not core_model'
349
343 -- label: Decoder-only Multi-Modal Models Test (Standard) # 22min
350 +- label: Multi-Modal Models Test (Standard) # 26min
351 #mirror_hardwares: [amd]
352 source_file_dependencies:
353 - vllm/
354 - tests/models/decoder_only/audio_language
355 - tests/models/decoder_only/vision_language
356 + - tests/models/embedding/vision_language
357 + - tests/models/encoder_decoder/vision_language
358 commands:
350 - - pytest -v -s models/decoder_only/audio_language -m core_model
351 - - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m core_model
352 - # No tests under this group for now
353 - # - pytest -v -s models/decoder_only/audio_language -m quant_model
354 - - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m quant_model
359 + - pytest -v -s models/decoder_only/audio_language -m 'core_model or quant_model'
360 + - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'core_model or quant_model'
361 + - pytest -v -s models/encoder_decoder/language -m core_model
362 + - pytest -v -s models/encoder_decoder/vision_language -m core_model
363
356 -- label: Decoder-only Multi-Modal Models Test (Extended) # 1h10m
364 +- label: Multi-Modal Models Test (Extended) # 1h15m
365 nightly: true
366 source_file_dependencies:
367 - vllm/
368 - tests/models/decoder_only/audio_language
369 - tests/models/decoder_only/vision_language
370 + - tests/models/embedding/vision_language
371 + - tests/models/encoder_decoder/vision_language
372 commands:
373 - pytest -v -s models/decoder_only/audio_language -m 'not core_model and not quant_model'
374 # HACK - run phi3v tests separately to sidestep this transformers bug
375 # https://github.com/huggingface/transformers/issues/34307
376 - pytest -v -s models/decoder_only/vision_language/test_phi3v.py
377 - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
368 -
369 -- label: Other Models Test # 20min
370 - #mirror_hardwares: [amd]
371 - source_file_dependencies:
372 - - vllm/
373 - - tests/models/embedding/language
374 - - tests/models/embedding/vision_language
375 - - tests/models/encoder_decoder/language
376 - - tests/models/encoder_decoder/vision_language
377 - commands:
378 - - pytest -v -s models/embedding/language
379 - - pytest -v -s models/embedding/vision_language
380 - - pytest -v -s models/encoder_decoder/language
381 - - pytest -v -s models/encoder_decoder/vision_language
378 + - pytest -v -s models/encoder_decoder/language -m 'not core_model'
379 + - pytest -v -s models/encoder_decoder/vision_language -m 'not core_model'
380
381 # This test is used only in PR development phase to test individual models and should never run on main
382 - label: Custom Models Test
docs/source/models/supported_models.rst
+9 -4
@@ -330,11 +330,16 @@ Text Embedding
330 - :code:`BAAI/bge-multilingual-gemma2`, etc.
331 -
332 - ✅︎
333 - * - :code:`MistralModel`
334 - - Mistral-based
333 + * - :code:`LlamaModel`, :code:`LlamaForCausalLM`, :code:`MistralModel`, etc.
334 + - Llama-based
335 - :code:`intfloat/e5-mistral-7b-instruct`, etc.
336 - ✅︎
337 - ✅︎
338 + * - :code:`Qwen2Model`, :code:`Qwen2ForCausalLM`
339 + - Qwen2-based
340 + - :code:`ssmits/Qwen2-7B-Instruct-embed-base`, :code:`Alibaba-NLP/gte-Qwen2-1.5B-instruct`, etc.
341 + - ✅︎
342 + - ✅︎
343
344 .. important::
345 Some model architectures support both generation and embedding tasks.
@@ -355,7 +360,7 @@ Reward Modeling
360 * - :code:`Qwen2ForRewardModel`
361 - Qwen2-based
362 - :code:`Qwen/Qwen2.5-Math-RM-72B`, etc.
358 - -
363 + - ✅︎
364 - ✅︎
365
366 .. note::
@@ -376,7 +381,7 @@ Classification
381 * - :code:`Qwen2ForSequenceClassification`
382 - Qwen2-based
383 - :code:`jason9693/Qwen2.5-1.5B-apeach`, etc.
379 - -
384 + - ✅︎
385 - ✅︎
386
387 .. note::
tests/models/decoder_only/language/test_jamba.py
+4 -14
@@ -33,6 +33,10 @@ def test_models(
33
34 with vllm_runner(model, dtype=dtype) as vllm_model:
35 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
36 + # This test is for verifying whether the model's extra_repr
37 + # can be printed correctly.
38 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
39 + model_runner.model)
40
41 for i in range(len(example_prompts)):
42 hf_output_ids, hf_output_str = hf_outputs[i]
@@ -293,17 +297,3 @@ def test_jamba_distributed_produces_identical_generation(
297 name_0="vllm_tp_1",
298 name_1="vllm_tp_2",
299 )
296 -
297 -
298 -@pytest.mark.parametrize("model", MODELS)
299 -@pytest.mark.parametrize("dtype", ["float"])
300 -def test_model_print(
301 - vllm_runner,
302 - model: str,
303 - dtype: str,
304 -) -> None:
305 - with vllm_runner(model, dtype=dtype) as vllm_model:
306 - # This test is for verifying whether the model's extra_repr
307 - # can be printed correctly.
308 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
309 - model_runner.model)
tests/models/decoder_only/language/test_mamba.py
+4 -14
@@ -51,6 +51,10 @@ def test_models(
51
52 with vllm_runner(model, dtype=dtype) as vllm_model:
53 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54 + # This test is for verifying whether the model's extra_repr
55 + # can be printed correctly.
56 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
57 + model_runner.model)
58
59 for i in range(len(example_prompts)):
60 hf_output_ids, hf_output_str = hf_outputs[i]
@@ -279,17 +283,3 @@ def test_state_cleanup(
283 except ValueError:
284 pytest.fail("Mamba inner state wasn't cleaned up between states, "
285 "could be related to finished_requests_ids")
282 -
283 -
284 -@pytest.mark.parametrize("model", MODELS)
285 -@pytest.mark.parametrize("dtype", ["float"])
286 -def test_model_print(
287 - vllm_runner,
288 - model: str,
289 - dtype: str,
290 -) -> None:
291 - with vllm_runner(model, dtype=dtype) as vllm_model:
292 - # This test is for verifying whether the model's extra_repr
293 - # can be printed correctly.
294 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
295 - model_runner.model)
tests/models/decoder_only/language/test_models.py
+43 -28
@@ -4,37 +4,52 @@ Run `pytest tests/models/test_models.py`.
4 """
5 import pytest
6
7 -from vllm.platforms import current_platform
8 -
7 from ...utils import check_logprobs_close
8
11 -MODELS = [
12 - "facebook/opt-125m", # opt
13 - "openai-community/gpt2", # gpt2
14 - # "Milos/slovak-gpt-j-405M", # gptj
15 - # "bigcode/tiny_starcoder_py", # gpt_bigcode
16 - # "EleutherAI/pythia-70m", # gpt_neox
17 - "bigscience/bloom-560m", # bloom - testing alibi slopes
18 - "microsoft/phi-2", # phi
19 - # "stabilityai/stablelm-3b-4e1t", # stablelm
20 - # "bigcode/starcoder2-3b", # starcoder2
21 - "google/gemma-1.1-2b-it", # gemma
22 - "Qwen/Qwen2.5-0.5B-Instruct", # qwen2
23 - "meta-llama/Llama-3.2-1B-Instruct", # llama
24 -]
25 -
26 -if not current_platform.is_cpu():
27 - MODELS += [
28 - # fused_moe which not supported on CPU
29 - "openbmb/MiniCPM3-4B",
30 - ]
31 -
32 -target_dtype = "half"
33 -
9
35 -@pytest.mark.core_model
36 -@pytest.mark.parametrize("model", MODELS)
37 -@pytest.mark.parametrize("dtype", [target_dtype])
10 +@pytest.mark.parametrize(
11 + "model",
12 + [
13 + pytest.param(
14 + "bigscience/bloom-560m", # bloom - testing alibi slopes
15 + marks=[pytest.mark.core_model, pytest.mark.cpu_model],
16 + ),
17 + pytest.param(
18 + "openai-community/gpt2", # gpt2
19 + marks=[pytest.mark.core_model, pytest.mark.cpu_model],
20 + ),
21 + pytest.param("Milos/slovak-gpt-j-405M"), # gptj
22 + pytest.param("bigcode/tiny_starcoder_py"), # gpt_bigcode
23 + pytest.param("EleutherAI/pythia-70m"), # gpt_neox
24 + pytest.param(
25 + "google/gemma-1.1-2b-it", # gemma
26 + marks=[pytest.mark.core_model, pytest.mark.cpu_model],
27 + ),
28 + pytest.param(
29 + "meta-llama/Llama-3.2-1B-Instruct", # llama
30 + marks=[pytest.mark.core_model, pytest.mark.cpu_model],
31 + ),
32 + pytest.param(
33 + "openbmb/MiniCPM3-4B",
34 + # fused_moe not supported on CPU
35 + marks=[pytest.mark.core_model],
36 + ),
37 + pytest.param(
38 + "facebook/opt-125m", # opt
39 + marks=[pytest.mark.core_model, pytest.mark.cpu_model],
40 + ),
41 + pytest.param(
42 + "microsoft/phi-2", # phi
43 + marks=[pytest.mark.core_model],
44 + ),
45 + pytest.param(
46 + "Qwen/Qwen2.5-0.5B-Instruct", # qwen2
47 + marks=[pytest.mark.core_model],
48 + ),
49 + pytest.param("stabilityai/stablelm-3b-4e1t"), # stablelm
50 + pytest.param("bigcode/starcoder2-3b"), # starcoder2
51 + ])
52 +@pytest.mark.parametrize("dtype", ["half"])
53 @pytest.mark.parametrize("max_tokens", [32])
54 @pytest.mark.parametrize("num_logprobs", [5])
55 def test_models(
tests/models/embedding/language/test_cls_models.py
+11 -19
@@ -9,10 +9,14 @@ import pytest
9 import torch
10 from transformers import AutoModelForSequenceClassification
11
12 -CLASSIFICATION_MODELS = ["jason9693/Qwen2.5-1.5B-apeach"]
12
14 -
15 -@pytest.mark.parametrize("model", CLASSIFICATION_MODELS)
13 +@pytest.mark.parametrize(
14 + "model",
15 + [
16 + pytest.param("jason9693/Qwen2.5-1.5B-apeach",
17 + marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
18 + ],
19 +)
20 @pytest.mark.parametrize("dtype", ["float"])
21 def test_classification_models(
22 hf_runner,
@@ -23,31 +27,19 @@ def test_classification_models(
27 ) -> None:
28 with vllm_runner(model, dtype=dtype) as vllm_model:
29 vllm_outputs = vllm_model.classify(example_prompts)
30 + # This test is for verifying whether the model's extra_repr
31 + # can be printed correctly.
32 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
33 + model_runner.model)
34
35 with hf_runner(model,
36 dtype=dtype,
37 auto_cls=AutoModelForSequenceClassification) as hf_model:
38 hf_outputs = hf_model.classify(example_prompts)
39
32 - print(hf_outputs, vllm_outputs)
33 -
40 # check logits difference
41 for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
42 hf_output = torch.tensor(hf_output)
43 vllm_output = torch.tensor(vllm_output)
44
45 assert torch.allclose(hf_output, vllm_output, 1e-3)
40 -
41 -
42 -@pytest.mark.parametrize("model", CLASSIFICATION_MODELS)
43 -@pytest.mark.parametrize("dtype", ["float"])
44 -def test_classification_model_print(
45 - vllm_runner,
46 - model: str,
47 - dtype: str,
48 -) -> None:
49 - with vllm_runner(model, dtype=dtype) as vllm_model:
50 - # This test is for verifying whether the model's extra_repr
51 - # can be printed correctly.
52 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
53 - model_runner.model)
tests/models/embedding/language/test_embedding.py
+22 -20
@@ -4,25 +4,25 @@ Run `pytest tests/models/embedding/language/test_embedding.py`.
4 """
5 import pytest
6
7 -from vllm.utils import current_platform
8 -
7 from ..utils import check_embeddings_close
8
11 -# Model, Guard
12 -MODELS = [
13 - "intfloat/e5-mistral-7b-instruct",
14 - "BAAI/bge-base-en-v1.5",
15 - "BAAI/bge-multilingual-gemma2",
16 - "intfloat/multilingual-e5-large",
17 -]
18 -
19 -ENCODER_ONLY = [
20 - "BAAI/bge-base-en-v1.5",
21 - "intfloat/multilingual-e5-large",
22 -]
9
24 -
25 -@pytest.mark.parametrize("model", MODELS)
10 +@pytest.mark.parametrize(
11 + "model",
12 + [
13 + # [Encoder-only]
14 + pytest.param("BAAI/bge-base-en-v1.5",
15 + marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
16 + pytest.param("intfloat/multilingual-e5-large"),
17 + # [Encoder-decoder]
18 + pytest.param("intfloat/e5-mistral-7b-instruct",
19 + marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
20 + pytest.param("BAAI/bge-multilingual-gemma2",
21 + marks=[pytest.mark.core_model]),
22 + pytest.param("ssmits/Qwen2-7B-Instruct-embed-base"),
23 + pytest.param("Alibaba-NLP/gte-Qwen2-1.5B-instruct"),
24 + ],
25 +)
26 @pytest.mark.parametrize("dtype", ["half"])
27 def test_models(
28 hf_runner,
@@ -31,9 +31,6 @@ def test_models(
31 model,
32 dtype: str,
33 ) -> None:
34 - if model not in ENCODER_ONLY and current_platform.is_cpu():
35 - pytest.skip("Skip large embedding models test on CPU.")
36 -
34 # The example_prompts has ending "\n", for example:
35 # "Write a short story about a robot that dreams for the first time.\n"
36 # sentence_transformers will strip the input texts, see:
@@ -46,8 +43,13 @@ def test_models(
43 is_sentence_transformer=True) as hf_model:
44 hf_outputs = hf_model.encode(example_prompts)
45
49 - with vllm_runner(model, dtype=dtype, max_model_len=None) as vllm_model:
46 + with vllm_runner(model, task="embedding", dtype=dtype,
47 + max_model_len=None) as vllm_model:
48 vllm_outputs = vllm_model.encode(example_prompts)
49 + # This test is for verifying whether the model's extra_repr
50 + # can be printed correctly.
51 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
52 + model_runner.model)
53
54 check_embeddings_close(
55 embeddings_0_lst=hf_outputs,
tests/models/embedding/vision_language/test_llava_next.py
+2
@@ -88,6 +88,7 @@ def _run_test(
88
89 @pytest.mark.skipif(transformers.__version__.startswith("4.46"),
90 reason="Model broken with changes in transformers 4.46")
91 +@pytest.mark.core_model
92 @pytest.mark.parametrize("model", MODELS)
93 @pytest.mark.parametrize("dtype", ["half"])
94 def test_models_text(
@@ -112,6 +113,7 @@ def test_models_text(
113
114
115 @large_gpu_test(min_gb=48)
116 +@pytest.mark.core_model
117 @pytest.mark.parametrize("model", MODELS)
118 @pytest.mark.parametrize("dtype", ["half"])
119 def test_models_image(
tests/models/embedding/vision_language/test_phi3v.py
+2
@@ -74,6 +74,7 @@ def _run_test(
74 )
75
76
77 +@pytest.mark.core_model
78 @pytest.mark.parametrize("model", MODELS)
79 @pytest.mark.parametrize("dtype", ["half"])
80 def test_models_text(
@@ -98,6 +99,7 @@ def test_models_text(
99
100
101 @large_gpu_test(min_gb=48)
102 +@pytest.mark.core_model
103 @pytest.mark.parametrize("model", MODELS)
104 @pytest.mark.parametrize("dtype", ["half"])
105 def test_models_image(
tests/models/encoder_decoder/language/test_bart.py
+8 -3
@@ -14,8 +14,6 @@ from ....conftest import (DecoderPromptType, ExplicitEncoderDecoderPrompt,
14 from ....utils import multi_gpu_test
15 from ...utils import check_logprobs_close
16
17 -MODELS = ["facebook/bart-base", "facebook/bart-large-cnn"]
18 -
17
18 def vllm_to_hf_output(
19 vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
@@ -170,7 +168,14 @@ def run_test(
168 )
169
170
173 -@pytest.mark.parametrize("model", MODELS)
171 +@pytest.mark.parametrize(
172 + "model",
173 + [
174 + pytest.param("facebook/bart-base",
175 + marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
176 + pytest.param("facebook/bart-large-cnn"),
177 + ],
178 +)
179 @pytest.mark.parametrize("dtype", ["float", "bfloat16"])
180 @pytest.mark.parametrize("max_tokens", [64])
181 @pytest.mark.parametrize("num_logprobs", [5])
tests/models/encoder_decoder/vision_language/test_mllama.py
+3
@@ -233,6 +233,7 @@ def clear_cache():
233
234
235 @large_gpu_test(min_gb=48)
236 +@pytest.mark.core_model
237 @pytest.mark.parametrize("model", models)
238 @pytest.mark.parametrize(
239 "sizes",
@@ -278,6 +279,7 @@ def test_models_single_leading_image(hf_runner, vllm_runner, image_assets,
279
280
281 @large_gpu_test(min_gb=48)
282 +@pytest.mark.core_model
283 @pytest.mark.parametrize("model", models)
284 @pytest.mark.parametrize("dtype", ["bfloat16"])
285 @pytest.mark.parametrize("max_tokens", [128])
@@ -326,6 +328,7 @@ def test_models_multi_leading_images(hf_runner, vllm_runner, image_assets,
328
329
330 @large_gpu_test(min_gb=48)
331 +@pytest.mark.core_model
332 @pytest.mark.parametrize("model", models)
333 @pytest.mark.parametrize("dtype", ["bfloat16"])
334 @pytest.mark.parametrize("max_tokens", [128])
tests/models/registry.py
+4
@@ -129,9 +129,13 @@ _EMBEDDING_EXAMPLE_MODELS = {
129 # [Text-only]
130 "BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
131 "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
132 + "LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
133 "MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
134 + "Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
135 "Qwen2ForRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-RM-72B"),
136 "Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"), # noqa: E501
137 + "RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2"), # noqa: E501
138 + "XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-large"),
139 # [Multimodal]
140 "LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"),
141 "Phi3VForCausalLM": _HfExamplesInfo("TIGER-Lab/VLM2Vec-Full",
tests/models/test_registry.py
+2 -2
@@ -77,8 +77,8 @@ def test_registry_is_pp(model_arch, is_pp, init_cuda):
77
78
79 def test_hf_registry_coverage():
80 - untested_archs = (HF_EXAMPLE_MODELS.get_supported_archs() -
81 - set(ModelRegistry.get_supported_archs()))
80 + untested_archs = (ModelRegistry.get_supported_archs() -
81 + HF_EXAMPLE_MODELS.get_supported_archs())
82
83 assert not untested_archs, (
84 "Please add the following architectures to "
vllm/model_executor/models/qwen2.py
+99 -13
@@ -37,6 +37,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37 QKVParallelLinear,
38 RowParallelLinear)
39 from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 +from vllm.model_executor.layers.pooler import Pooler, PoolingType
41 from vllm.model_executor.layers.quantization import QuantizationConfig
42 from vllm.model_executor.layers.rotary_embedding import get_rope
43 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -44,8 +45,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
45 ParallelLMHead, VocabParallelEmbedding)
46 from vllm.model_executor.model_loader.weight_utils import (
47 default_weight_loader, maybe_remap_kv_scale_name)
48 +from vllm.model_executor.pooling_metadata import PoolingMetadata
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
48 -from vllm.sequence import IntermediateTensors
50 +from vllm.sequence import IntermediateTensors, PoolerOutput
51
52 from .interfaces import SupportsLoRA, SupportsPP
53 from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
@@ -247,6 +249,18 @@ class Qwen2Model(nn.Module):
249 cache_config = vllm_config.cache_config
250 quant_config = vllm_config.quant_config
251
252 + # TODO (@robertgshaw2): see if this can be moved out
253 + if (cache_config.sliding_window is not None
254 + and hasattr(config, "max_window_layers")):
255 + raise ValueError("Sliding window for some but all layers is not "
256 + "supported. This model uses sliding window "
257 + "but `max_window_layers` = {} is less than "
258 + "`num_hidden_layers` = {}. Please open an issue "
259 + "to discuss this feature.".format(
260 + config.max_window_layers,
261 + config.num_hidden_layers,
262 + ))
263 +
264 self.config = config
265 self.padding_idx = config.pad_token_id
266 self.vocab_size = config.vocab_size
@@ -405,20 +419,9 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
419 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
420 super().__init__()
421 config = vllm_config.model_config.hf_config
408 - cache_config = vllm_config.cache_config
422 quant_config = vllm_config.quant_config
423 lora_config = vllm_config.lora_config
411 - # TODO (@robertgshaw2): see if this can be moved out
412 - if (cache_config.sliding_window is not None
413 - and hasattr(config, "max_window_layers")):
414 - raise ValueError("Sliding window for some but all layers is not "
415 - "supported. This model uses sliding window "
416 - "but `max_window_layers` = {} is less than "
417 - "`num_hidden_layers` = {}. Please open an issue "
418 - "to discuss this feature.".format(
419 - config.max_window_layers,
420 - config.num_hidden_layers,
421 - ))
424 + pooler_config = vllm_config.model_config.pooler_config
425
426 self.config = config
427 self.lora_config = lora_config
@@ -438,6 +441,15 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
441
442 self.logits_processor = LogitsProcessor(config.vocab_size)
443 self.sampler = get_sampler()
444 +
445 + # The same model class supports both language generation and embedding
446 + # because the architecture name is the same
447 + self._pooler = Pooler.from_config_with_defaults(
448 + pooler_config,
449 + pooling_type=PoolingType.LAST,
450 + normalize=True,
451 + softmax=False)
452 +
453 self.make_empty_intermediate_tensors = (
454 self.model.make_empty_intermediate_tensors)
455
@@ -475,6 +487,13 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
487 next_tokens = self.sampler(logits, sampling_metadata)
488 return next_tokens
489
490 + def pooler(
491 + self,
492 + hidden_states: torch.Tensor,
493 + pooling_metadata: PoolingMetadata,
494 + ) -> Optional[PoolerOutput]:
495 + return self._pooler(hidden_states, pooling_metadata)
496 +
497 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
498 loader = AutoWeightsLoader(
499 self,
@@ -482,3 +501,70 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
501 if self.config.tie_word_embeddings else None),
502 )
503 loader.load_weights(weights)
504 +
505 +
506 +class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
507 + packed_modules_mapping = {
508 + "qkv_proj": [
509 + "q_proj",
510 + "k_proj",
511 + "v_proj",
512 + ],
513 + "gate_up_proj": [
514 + "gate_proj",
515 + "up_proj",
516 + ],
517 + }
518 +
519 + # LoRA specific attributes
520 + supported_lora_modules = [
521 + "qkv_proj",
522 + "o_proj",
523 + "gate_up_proj",
524 + "down_proj",
525 + ]
526 + embedding_modules = {}
527 + embedding_padding_modules = []
528 +
529 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
530 + super().__init__()
531 + config = vllm_config.model_config.hf_config
532 + quant_config = vllm_config.quant_config
533 + lora_config = vllm_config.lora_config
534 + pooler_config = vllm_config.model_config.pooler_config
535 +
536 + self.config = config
537 + self.lora_config = lora_config
538 +
539 + self.quant_config = quant_config
540 + self.model = Qwen2Model(vllm_config=vllm_config,
541 + prefix=maybe_prefix(prefix, "model"))
542 +
543 + self._pooler = Pooler.from_config_with_defaults(
544 + pooler_config,
545 + pooling_type=PoolingType.MEAN,
546 + normalize=True,
547 + softmax=False)
548 +
549 + def forward(
550 + self,
551 + input_ids: torch.Tensor,
552 + positions: torch.Tensor,
553 + kv_caches: List[torch.Tensor],
554 + attn_metadata: AttentionMetadata,
555 + intermediate_tensors: Optional[IntermediateTensors] = None,
556 + ) -> torch.Tensor:
557 + return self.model(input_ids, positions, kv_caches, attn_metadata,
558 + intermediate_tensors)
559 +
560 + def pooler(
561 + self,
562 + hidden_states: torch.Tensor,
563 + pooling_metadata: PoolingMetadata,
564 + ) -> Optional[PoolerOutput]:
565 + return self._pooler(hidden_states, pooling_metadata)
566 +
567 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
568 + loader = AutoWeightsLoader(self,
569 + ignore_unexpected_prefixes=["lm_head."])
570 + loader.load_weights(weights)
vllm/model_executor/models/qwen2_cls.py
+2 -13
@@ -17,10 +17,11 @@ from vllm.model_executor.models.qwen2 import Qwen2Model
17 from vllm.model_executor.pooling_metadata import PoolingMetadata
18 from vllm.sequence import IntermediateTensors, PoolerOutput
19
20 +from .interfaces import SupportsLoRA, SupportsPP
21 from .utils import AutoWeightsLoader, maybe_prefix
22
23
23 -class Qwen2ForSequenceClassification(nn.Module):
24 +class Qwen2ForSequenceClassification(nn.Module, SupportsLoRA, SupportsPP):
25 packed_modules_mapping = {
26 "qkv_proj": [
27 "q_proj",
@@ -46,21 +47,9 @@ class Qwen2ForSequenceClassification(nn.Module):
47 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
48 super().__init__()
49 config = vllm_config.model_config.hf_config
49 - cache_config = vllm_config.cache_config
50 quant_config = vllm_config.quant_config
51 lora_config = vllm_config.lora_config
52 pooler_config = vllm_config.model_config.pooler_config
53 - # TODO (@robertgshaw2): see if this can be moved out
54 - if (cache_config.sliding_window is not None
55 - and hasattr(config, "max_window_layers")):
56 - raise ValueError("Sliding window for some but all layers is not "
57 - "supported. This model uses sliding window "
58 - "but `max_window_layers` = {} is less than "
59 - "`num_hidden_layers` = {}. Please open an issue "
60 - "to discuss this feature.".format(
61 - config.max_window_layers,
62 - config.num_hidden_layers,
63 - ))
53
54 self.config = config
55 self.lora_config = lora_config
vllm/model_executor/models/qwen2_rm.py
+2 -14
@@ -16,7 +16,7 @@ from vllm.model_executor.layers.pooler import Pooler, PoolingType
16 from vllm.model_executor.pooling_metadata import PoolingMetadata
17 from vllm.sequence import IntermediateTensors, PoolerOutput
18
19 -from .interfaces import SupportsPP
19 +from .interfaces import SupportsLoRA, SupportsPP
20 from .qwen2 import Qwen2Model
21 from .utils import AutoWeightsLoader, maybe_prefix
22
@@ -32,7 +32,7 @@ class ReLU(nn.Module):
32 return self.activation(input)
33
34
35 -class Qwen2ForRewardModel(nn.Module, SupportsPP):
35 +class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
36 packed_modules_mapping = {
37 "qkv_proj": [
38 "q_proj",
@@ -58,21 +58,9 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
58 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
59 super().__init__()
60 config = vllm_config.model_config.hf_config
61 - cache_config = vllm_config.cache_config
61 quant_config = vllm_config.quant_config
62 lora_config = vllm_config.lora_config
63 pooler_config = vllm_config.model_config.pooler_config
65 - # TODO (@robertgshaw2): see if this can be moved out
66 - if (cache_config.sliding_window is not None
67 - and hasattr(config, "max_window_layers")):
68 - raise ValueError("Sliding window for some but all layers is not "
69 - "supported. This model uses sliding window "
70 - "but `max_window_layers` = {} is less than "
71 - "`num_hidden_layers` = {}. Please open an issue "
72 - "to discuss this feature.".format(
73 - config.max_window_layers,
74 - config.num_hidden_layers,
75 - ))
64
65 self.config = config
66 self.lora_config = lora_config
vllm/model_executor/models/registry.py
+6 -3
@@ -11,7 +11,8 @@ import tempfile
11 from abc import ABC, abstractmethod
12 from dataclasses import dataclass, field
13 from functools import lru_cache
14 -from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union
14 +from typing import (AbstractSet, Callable, Dict, List, Optional, Tuple, Type,
15 + TypeVar, Union)
16
17 import cloudpickle
18 import torch.nn as nn
@@ -110,6 +111,8 @@ _EMBEDDING_MODELS = {
111 },
112 "MistralModel": ("llama", "LlamaEmbeddingModel"),
113 "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
114 + "Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
115 + "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
116 "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
117 "Qwen2ForSequenceClassification": ("qwen2_cls", "Qwen2ForSequenceClassification"), # noqa: E501
118 # [Multimodal]
@@ -301,8 +304,8 @@ class _ModelRegistry:
304 # Keyed by model_arch
305 models: Dict[str, _BaseRegisteredModel] = field(default_factory=dict)
306
304 - def get_supported_archs(self) -> List[str]:
305 - return list(self.models.keys())
307 + def get_supported_archs(self) -> AbstractSet[str]:
308 + return self.models.keys()
309
310 def register_model(
311 self,