A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Split up pooling tasks (#10820)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 11, 2024 at 17:28 UTC
8f10d5e3930f05c2057a831cd80ba24c52b8ceef
27 files changed
+527
-168
docs/source/index.rst
+2
@@ -94,6 +94,8 @@ Documentation
94
:caption: Models
95
96
models/supported_models
97
+ models/generative_models
98
+ models/pooling_models
99
models/adding_model
100
models/enabling_multimodal_inputs
101
docs/source/models/generative_models.rst
new
+146
@@ -0,0 +1,146 @@
1
+.. _generative_models:
2
+
3
+Generative Models
4
+=================
5
+
6
+vLLM provides first-class support for generative models, which covers most of LLMs.
7
+
8
+In vLLM, generative models implement the :class:`~vllm.model_executor.models.VllmModelForTextGeneration` interface.
9
+Based on the final hidden states of the input, these models output log probabilities of the tokens to generate,
10
+which are then passed through :class:`~vllm.model_executor.layers.Sampler` to obtain the final text.
11
+
12
+Offline Inference
13
+-----------------
14
+
15
+The :class:`~vllm.LLM` class provides various methods for offline inference.
16
+See :ref:`Engine Arguments <engine_args>` for a list of options when initializing the model.
17
+
18
+For generative models, the only supported :code:`task` option is :code:`"generate"`.
19
+Usually, this is automatically inferred so you don't have to specify it.
20
+
21
+``LLM.generate``
22
+^^^^^^^^^^^^^^^^
23
+
24
+The :class:`~vllm.LLM.generate` method is available to all generative models in vLLM.
25
+It is similar to `its counterpart in HF Transformers <https://huggingface.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate>`__,
26
+except that tokenization and detokenization are also performed automatically.
27
+
28
+.. code-block:: python
29
+
30
+ llm = LLM(model="facebook/opt-125m")
31
+ outputs = llm.generate("Hello, my name is")
32
+
33
+ for output in outputs:
34
+ prompt = output.prompt
35
+ generated_text = output.outputs[0].text
36
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
37
+
38
+You can optionally control the language generation by passing :class:`~vllm.SamplingParams`.
39
+For example, you can use greedy sampling by setting :code:`temperature=0`:
40
+
41
+.. code-block:: python
42
+
43
+ llm = LLM(model="facebook/opt-125m")
44
+ params = SamplingParams(temperature=0)
45
+ outputs = llm.generate("Hello, my name is", params)
46
+
47
+ for output in outputs:
48
+ prompt = output.prompt
49
+ generated_text = output.outputs[0].text
50
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
51
+
52
+A code example can be found in `examples/offline_inference.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference.py>`_.
53
+
54
+``LLM.beam_search``
55
+^^^^^^^^^^^^^^^^^^^
56
+
57
+The :class:`~vllm.LLM.beam_search` method implements `beam search <https://huggingface.co/docs/transformers/en/generation_strategies#beam-search-decoding>`__ on top of :class:`~vllm.LLM.generate`.
58
+For example, to search using 5 beams and output at most 50 tokens:
59
+
60
+.. code-block:: python
61
+
62
+ llm = LLM(model="facebook/opt-125m")
63
+ params = BeamSearchParams(beam_width=5, max_tokens=50)
64
+ outputs = llm.generate("Hello, my name is", params)
65
+
66
+ for output in outputs:
67
+ prompt = output.prompt
68
+ generated_text = output.outputs[0].text
69
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
70
+
71
+``LLM.chat``
72
+^^^^^^^^^^^^
73
+
74
+The :class:`~vllm.LLM.chat` method implements chat functionality on top of :class:`~vllm.LLM.generate`.
75
+In particular, it accepts input similar to `OpenAI Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`__
76
+and automatically applies the model's `chat template <https://huggingface.co/docs/transformers/en/chat_templating>`__ to format the prompt.
77
+
78
+.. important::
79
+
80
+ In general, only instruction-tuned models have a chat template.
81
+ Base models may perform poorly as they are not trained to respond to the chat conversation.
82
+
83
+.. code-block:: python
84
+
85
+ llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
86
+ conversation = [
87
+ {
88
+ "role": "system",
89
+ "content": "You are a helpful assistant"
90
+ },
91
+ {
92
+ "role": "user",
93
+ "content": "Hello"
94
+ },
95
+ {
96
+ "role": "assistant",
97
+ "content": "Hello! How can I assist you today?"
98
+ },
99
+ {
100
+ "role": "user",
101
+ "content": "Write an essay about the importance of higher education.",
102
+ },
103
+ ]
104
+ outputs = llm.chat(conversation)
105
+
106
+ for output in outputs:
107
+ prompt = output.prompt
108
+ generated_text = output.outputs[0].text
109
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
110
+
111
+A code example can be found in `examples/offline_inference_chat.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_chat.py>`_.
112
+
113
+If the model doesn't have a chat template or you want to specify another one,
114
+you can explicitly pass a chat template:
115
+
116
+.. code-block:: python
117
+
118
+ from vllm.entrypoints.chat_utils import load_chat_template
119
+
120
+ # You can find a list of existing chat templates under `examples/`
121
+ custom_template = load_chat_template(chat_template="<path_to_template>")
122
+ print("Loaded chat template:", custom_template)
123
+
124
+ outputs = llm.chat(conversation, chat_template=custom_template)
125
+
126
+Online Inference
127
+----------------
128
+
129
+Our `OpenAI Compatible Server <../serving/openai_compatible_server>`__ can be used for online inference.
130
+Please click on the above link for more details on how to launch the server.
131
+
132
+Completions API
133
+^^^^^^^^^^^^^^^
134
+
135
+Our Completions API is similar to ``LLM.generate`` but only accepts text.
136
+It is compatible with `OpenAI Completions API <https://platform.openai.com/docs/api-reference/completions>`__
137
+so that you can use OpenAI client to interact with it.
138
+A code example can be found in `examples/openai_completion_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_completion_client.py>`_.
139
+
140
+Chat API
141
+^^^^^^^^
142
+
143
+Our Chat API is similar to ``LLM.chat``, accepting both text and :ref:`multi-modal inputs <multimodal_inputs>`.
144
+It is compatible with `OpenAI Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`__
145
+so that you can use OpenAI client to interact with it.
146
+A code example can be found in `examples/openai_chat_completion_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_client.py>`_.
docs/source/models/pooling_models.rst
new
+99
@@ -0,0 +1,99 @@
1
+.. _pooling_models:
2
+
3
+Pooling Models
4
+==============
5
+
6
+vLLM also supports pooling models, including embedding, reranking and reward models.
7
+
8
+In vLLM, pooling models implement the :class:`~vllm.model_executor.models.VllmModelForPooling` interface.
9
+These models use a :class:`~vllm.model_executor.layers.Pooler` to aggregate the final hidden states of the input
10
+before returning them.
11
+
12
+.. note::
13
+
14
+ We currently support pooling models primarily as a matter of convenience.
15
+ As shown in the :ref:`Compatibility Matrix <compatibility_matrix>`, most vLLM features are not applicable to
16
+ pooling models as they only work on the generation or decode stage, so performance may not improve as much.
17
+
18
+Offline Inference
19
+-----------------
20
+
21
+The :class:`~vllm.LLM` class provides various methods for offline inference.
22
+See :ref:`Engine Arguments <engine_args>` for a list of options when initializing the model.
23
+
24
+For pooling models, we support the following :code:`task` options:
25
+
26
+- Embedding (:code:`"embed"` / :code:`"embedding"`)
27
+- Classification (:code:`"classify"`)
28
+- Sentence Pair Scoring (:code:`"score"`)
29
+- Reward Modeling (:code:`"reward"`)
30
+
31
+The selected task determines the default :class:`~vllm.model_executor.layers.Pooler` that is used:
32
+
33
+- Embedding: Extract only the hidden states corresponding to the last token, and apply normalization.
34
+- Classification: Extract only the hidden states corresponding to the last token, and apply softmax.
35
+- Sentence Pair Scoring: Extract only the hidden states corresponding to the last token, and apply softmax.
36
+- Reward Modeling: Extract all of the hidden states and return them directly.
37
+
38
+When loading `Sentence Transformers <https://huggingface.co/sentence-transformers>`__ models,
39
+we attempt to override the default pooler based on its Sentence Transformers configuration file (:code:`modules.json`).
40
+
41
+You can customize the model's pooling method via the :code:`override_pooler_config` option,
42
+which takes priority over both the model's and Sentence Transformers's defaults.
43
+
44
+``LLM.encode``
45
+^^^^^^^^^^^^^^
46
+
47
+The :class:`~vllm.LLM.encode` method is available to all pooling models in vLLM.
48
+It returns the aggregated hidden states directly.
49
+
50
+.. code-block:: python
51
+
52
+ llm = LLM(model="intfloat/e5-mistral-7b-instruct", task="embed")
53
+ outputs = llm.encode("Hello, my name is")
54
+
55
+ outputs = model.encode(prompts)
56
+ for output in outputs:
57
+ embeddings = output.outputs.embedding
58
+ print(f"Prompt: {prompt!r}, Embeddings (size={len(embeddings)}: {embeddings!r}")
59
+
60
+A code example can be found in `examples/offline_inference_embedding.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_embedding.py>`_.
61
+
62
+``LLM.score``
63
+^^^^^^^^^^^^^
64
+
65
+The :class:`~vllm.LLM.score` method outputs similarity scores between sentence pairs.
66
+It is primarily designed for `cross-encoder models <https://www.sbert.net/examples/applications/cross-encoder/README.html>`__.
67
+These types of models serve as rerankers between candidate query-document pairs in RAG systems.
68
+
69
+.. note::
70
+
71
+ vLLM can only perform the model inference component (e.g. embedding, reranking) of RAG.
72
+ To handle RAG at a higher level, you should use integration frameworks such as `LangChain <https://github.com/langchain-ai/langchain>`_.
73
+
74
+You can use `these tests <https://github.com/vllm-project/vllm/blob/main/tests/models/embedding/language/test_scoring.py>`_ as reference.
75
+
76
+Online Inference
77
+----------------
78
+
79
+Our `OpenAI Compatible Server <../serving/openai_compatible_server>`__ can be used for online inference.
80
+Please click on the above link for more details on how to launch the server.
81
+
82
+Embeddings API
83
+^^^^^^^^^^^^^^
84
+
85
+Our Embeddings API is similar to ``LLM.encode``, accepting both text and :ref:`multi-modal inputs <multimodal_inputs>`.
86
+
87
+The text-only API is compatible with `OpenAI Embeddings API <https://platform.openai.com/docs/api-reference/embeddings>`__
88
+so that you can use OpenAI client to interact with it.
89
+A code example can be found in `examples/openai_embedding_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_embedding_client.py>`_.
90
+
91
+The multi-modal API is an extension of the `OpenAI Embeddings API <https://platform.openai.com/docs/api-reference/embeddings>`__
92
+that incorporates `OpenAI Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`__,
93
+so it is not part of the OpenAI standard. Please see :ref:`this page <multimodal_inputs>` for more details on how to use it.
94
+
95
+Score API
96
+^^^^^^^^^
97
+
98
+Our Score API is similar to ``LLM.score``.
99
+Please see `this page <../serving/openai_compatible_server.html#score-api-for-cross-encoder-models>`__ for more details on how to use it.
docs/source/models/supported_models.rst
+102
-55
@@ -3,11 +3,21 @@
3
Supported Models
4
================
5
6
-vLLM supports a variety of generative and embedding models from `HuggingFace (HF) Transformers <https://huggingface.co/models>`_.
7
-This page lists the model architectures that are currently supported by vLLM.
6
+vLLM supports generative and pooling models across various tasks.
7
+If a model supports more than one task, you can set the task via the :code:`--task` argument.
8
+
9
+For each task, we list the model architectures that have been implemented in vLLM.
10
Alongside each architecture, we include some popular models that use it.
11
10
-For other models, you can check the :code:`config.json` file inside the model repository.
12
+Loading a Model
13
+^^^^^^^^^^^^^^^
14
+
15
+HuggingFace Hub
16
++++++++++++++++
17
+
18
+By default, vLLM loads models from `HuggingFace (HF) Hub <https://huggingface.co/models>`_.
19
+
20
+To determine whether a given model is supported, you can check the :code:`config.json` file inside the HF repository.
21
If the :code:`"architectures"` field contains a model architecture listed below, then it should be supported in theory.
22
23
.. tip::
@@ -17,38 +27,57 @@ If the :code:`"architectures"` field contains a model architecture listed below,
27
28
from vllm import LLM
29
20
- llm = LLM(model=...) # Name or path of your model
30
+ # For generative models (task=generate) only
31
+ llm = LLM(model=..., task="generate") # Name or path of your model
32
output = llm.generate("Hello, my name is")
33
print(output)
34
24
- If vLLM successfully generates text, it indicates that your model is supported.
35
+ # For pooling models (task={embed,classify,reward}) only
36
+ llm = LLM(model=..., task="embed") # Name or path of your model
37
+ output = llm.encode("Hello, my name is")
38
+ print(output)
39
+
40
+ If vLLM successfully returns text (for generative models) or hidden states (for pooling models), it indicates that your model is supported.
41
42
Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` and :ref:`Enabling Multimodal Inputs <enabling_multimodal_inputs>`
43
for instructions on how to implement your model in vLLM.
44
Alternatively, you can `open an issue on GitHub <https://github.com/vllm-project/vllm/issues/new/choose>`_ to request vLLM support.
45
30
-.. note::
31
- To use models from `ModelScope <https://www.modelscope.cn>`_ instead of HuggingFace Hub, set an environment variable:
46
+ModelScope
47
+++++++++++
48
33
- .. code-block:: shell
49
+To use models from `ModelScope <https://www.modelscope.cn>`_ instead of HuggingFace Hub, set an environment variable:
50
35
- $ export VLLM_USE_MODELSCOPE=True
51
+.. code-block:: shell
52
37
- And use with :code:`trust_remote_code=True`.
53
+ $ export VLLM_USE_MODELSCOPE=True
54
39
- .. code-block:: python
55
+And use with :code:`trust_remote_code=True`.
56
41
- from vllm import LLM
57
+.. code-block:: python
58
43
- llm = LLM(model=..., revision=..., trust_remote_code=True) # Name or path of your model
44
- output = llm.generate("Hello, my name is")
45
- print(output)
59
+ from vllm import LLM
60
+
61
+ llm = LLM(model=..., revision=..., task=..., trust_remote_code=True)
62
47
-Text-only Language Models
48
-^^^^^^^^^^^^^^^^^^^^^^^^^
63
+ # For generative models (task=generate) only
64
+ output = llm.generate("Hello, my name is")
65
+ print(output)
66
50
-Text Generation
51
----------------
67
+ # For pooling models (task={embed,classify,reward}) only
68
+ output = llm.encode("Hello, my name is")
69
+ print(output)
70
+
71
+List of Text-only Language Models
72
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
73
+
74
+Generative Models
75
++++++++++++++++++
76
+
77
+See :ref:`this page <generative_models>` for more information on how to use generative models.
78
+
79
+Text Generation (``--task generate``)
80
+-------------------------------------
81
82
.. list-table::
83
:widths: 25 25 50 5 5
@@ -328,8 +357,24 @@ Text Generation
357
.. note::
358
Currently, the ROCm version of vLLM supports Mistral and Mixtral only for context lengths up to 4096.
359
331
-Text Embedding
332
---------------
360
+Pooling Models
361
+++++++++++++++
362
+
363
+See :ref:`this page <pooling_models>` for more information on how to use pooling models.
364
+
365
+.. important::
366
+ Since some model architectures support both generative and pooling tasks,
367
+ you should explicitly specify the task type to ensure that the model is used in pooling mode instead of generative mode.
368
+
369
+Text Embedding (``--task embed``)
370
+---------------------------------
371
+
372
+Any text generation model can be converted into an embedding model by passing :code:`--task embed`.
373
+
374
+.. note::
375
+ To get the best results, you should use pooling models that are specifically trained as such.
376
+
377
+The following table lists those that are tested in vLLM.
378
379
.. list-table::
380
:widths: 25 25 50 5 5
@@ -371,13 +416,6 @@ Text Embedding
416
-
417
-
418
374
-.. important::
375
- Some model architectures support both generation and embedding tasks.
376
- In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
377
-
378
-.. tip::
379
- You can override the model's pooling method by passing :code:`--override-pooler-config`.
380
-
419
.. note::
420
:code:`ssmits/Qwen2-7B-Instruct-embed-base` has an improperly defined Sentence Transformers config.
421
You should manually set mean pooling by passing :code:`--override-pooler-config '{"pooling_type": "MEAN"}'`.
@@ -389,8 +427,8 @@ Text Embedding
427
On the other hand, its 1.5B variant (:code:`Alibaba-NLP/gte-Qwen2-1.5B-instruct`) uses causal attention
428
despite being described otherwise on its model card.
429
392
-Reward Modeling
393
----------------
430
+Reward Modeling (``--task reward``)
431
+-----------------------------------
432
433
.. list-table::
434
:widths: 25 25 50 5 5
@@ -416,11 +454,8 @@ Reward Modeling
454
For process-supervised reward models such as :code:`peiyi9979/math-shepherd-mistral-7b-prm`, the pooling config should be set explicitly,
455
e.g.: :code:`--override-pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
456
419
-.. note::
420
- As an interim measure, these models are supported in both offline and online inference via Embeddings API.
421
-
422
-Classification
423
----------------
457
+Classification (``--task classify``)
458
+------------------------------------
459
460
.. list-table::
461
:widths: 25 25 50 5 5
@@ -437,11 +472,8 @@ Classification
472
- ✅︎
473
- ✅︎
474
440
-.. note::
441
- As an interim measure, these models are supported in both offline and online inference via Embeddings API.
442
-
443
-Sentence Pair Scoring
444
----------------------
475
+Sentence Pair Scoring (``--task score``)
476
+----------------------------------------
477
478
.. list-table::
479
:widths: 25 25 50 5 5
@@ -468,13 +500,10 @@ Sentence Pair Scoring
500
-
501
-
502
471
-.. note::
472
- These models are supported in both offline and online inference via Score API.
473
-
503
.. _supported_mm_models:
504
476
-Multimodal Language Models
477
-^^^^^^^^^^^^^^^^^^^^^^^^^^
505
+List of Multimodal Language Models
506
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
507
508
The following modalities are supported depending on the model:
509
@@ -491,8 +520,15 @@ On the other hand, modalities separated by :code:`/` are mutually exclusive.
520
521
- e.g.: :code:`T / I` means that the model supports text-only and image-only inputs, but not text-with-image inputs.
522
494
-Text Generation
495
----------------
523
+See :ref:`this page <multimodal_inputs>` on how to pass multi-modal inputs to the model.
524
+
525
+Generative Models
526
++++++++++++++++++
527
+
528
+See :ref:`this page <generative_models>` for more information on how to use generative models.
529
+
530
+Text Generation (``--task generate``)
531
+-------------------------------------
532
533
.. list-table::
534
:widths: 25 25 15 20 5 5 5
@@ -696,8 +732,24 @@ Text Generation
732
The official :code:`openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
733
For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
734
699
-Multimodal Embedding
700
---------------------
735
+Pooling Models
736
+++++++++++++++
737
+
738
+See :ref:`this page <pooling_models>` for more information on how to use pooling models.
739
+
740
+.. important::
741
+ Since some model architectures support both generative and pooling tasks,
742
+ you should explicitly specify the task type to ensure that the model is used in pooling mode instead of generative mode.
743
+
744
+Text Embedding (``--task embed``)
745
+---------------------------------
746
+
747
+Any text generation model can be converted into an embedding model by passing :code:`--task embed`.
748
+
749
+.. note::
750
+ To get the best results, you should use pooling models that are specifically trained as such.
751
+
752
+The following table lists those that are tested in vLLM.
753
754
.. list-table::
755
:widths: 25 25 15 25 5 5
@@ -728,12 +780,7 @@ Multimodal Embedding
780
-
781
- ✅︎
782
731
-.. important::
732
- Some model architectures support both generation and embedding tasks.
733
- In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
734
-
735
-.. tip::
736
- You can override the model's pooling method by passing :code:`--override-pooler-config`.
783
+----
784
785
Model Support Policy
786
=====================
docs/source/usage/compatibility_matrix.rst
+6
-6
@@ -39,13 +39,13 @@ Feature x Feature
39
- :abbr:`prmpt adptr (Prompt Adapter)`
40
- :ref:`SD <spec_decode>`
41
- CUDA graph
42
- - :abbr:`emd (Embedding Models)`
42
+ - :abbr:`pooling (Pooling Models)`
43
- :abbr:`enc-dec (Encoder-Decoder Models)`
44
- :abbr:`logP (Logprobs)`
45
- :abbr:`prmpt logP (Prompt Logprobs)`
46
- :abbr:`async output (Async Output Processing)`
47
- multi-step
48
- - :abbr:`mm (Multimodal)`
48
+ - :abbr:`mm (Multimodal Inputs)`
49
- best-of
50
- beam-search
51
- :abbr:`guided dec (Guided Decoding)`
@@ -151,7 +151,7 @@ Feature x Feature
151
-
152
-
153
-
154
- * - :abbr:`emd (Embedding Models)`
154
+ * - :abbr:`pooling (Pooling Models)`
155
- ✗
156
- ✗
157
- ✗
@@ -253,7 +253,7 @@ Feature x Feature
253
-
254
-
255
-
256
- * - :abbr:`mm (Multimodal)`
256
+ * - :abbr:`mm (Multimodal Inputs)`
257
- ✅
258
- `✗ <https://github.com/vllm-project/vllm/pull/8348>`__
259
- `✗ <https://github.com/vllm-project/vllm/pull/7199>`__
@@ -386,7 +386,7 @@ Feature x Hardware
386
- ✅
387
- ✗
388
- ✅
389
- * - :abbr:`emd (Embedding Models)`
389
+ * - :abbr:`pooling (Pooling Models)`
390
- ✅
391
- ✅
392
- ✅
@@ -402,7 +402,7 @@ Feature x Hardware
402
- ✅
403
- ✅
404
- ✗
405
- * - :abbr:`mm (Multimodal)`
405
+ * - :abbr:`mm (Multimodal Inputs)`
406
- ✅
407
- ✅
408
- ✅
examples/offline_inference_embedding.py
+6
-1
@@ -9,7 +9,12 @@ prompts = [
9
]
10
11
# Create an LLM.
12
-model = LLM(model="intfloat/e5-mistral-7b-instruct", enforce_eager=True)
12
+model = LLM(
13
+ model="intfloat/e5-mistral-7b-instruct",
14
+ task="embed", # You should pass task="embed" for embedding models
15
+ enforce_eager=True,
16
+)
17
+
18
# Generate embedding. The output is a list of PoolingRequestOutputs.
19
outputs = model.encode(prompts)
20
# Print the outputs.
examples/offline_inference_vision_language_embedding.py
+2
-2
@@ -59,7 +59,7 @@ def run_e5_v(query: Query):
59
60
llm = LLM(
61
model="royokong/e5-v",
62
- task="embedding",
62
+ task="embed",
63
max_model_len=4096,
64
)
65
@@ -88,7 +88,7 @@ def run_vlm2vec(query: Query):
88
89
llm = LLM(
90
model="TIGER-Lab/VLM2Vec-Full",
91
- task="embedding",
91
+ task="embed",
92
trust_remote_code=True,
93
mm_processor_kwargs={"num_crops": 4},
94
)
tests/compile/test_basic_correctness.py
+2
-2
@@ -55,7 +55,7 @@ test_settings = [
55
# embedding model
56
TestSetting(
57
model="BAAI/bge-multilingual-gemma2",
58
- model_args=["--task", "embedding"],
58
+ model_args=["--task", "embed"],
59
pp_size=1,
60
tp_size=1,
61
attn_backend="FLASHINFER",
@@ -65,7 +65,7 @@ test_settings = [
65
# encoder-based embedding model (BERT)
66
TestSetting(
67
model="BAAI/bge-base-en-v1.5",
68
- model_args=["--task", "embedding"],
68
+ model_args=["--task", "embed"],
69
pp_size=1,
70
tp_size=1,
71
attn_backend="XFORMERS",
tests/core/test_scheduler_encoder_decoder.py
+1
-1
@@ -37,7 +37,7 @@ def test_scheduler_schedule_simple_encoder_decoder():
37
num_seq_group = 4
38
max_model_len = 16
39
scheduler_config = SchedulerConfig(
40
- task="generate",
40
+ "generate",
41
max_num_batched_tokens=64,
42
max_num_seqs=num_seq_group,
43
max_model_len=max_model_len,
tests/entrypoints/openai/test_vision_embedding.py
+1
-1
@@ -27,7 +27,7 @@ TEST_IMAGE_URLS = [
27
def server():
28
args = [
29
"--task",
30
- "embedding",
30
+ "embed",
31
"--dtype",
32
"bfloat16",
33
"--max-model-len",
tests/models/embedding/language/test_embedding.py
+1
-1
@@ -54,7 +54,7 @@ def test_models(
54
hf_outputs = hf_model.encode(example_prompts)
55
56
with vllm_runner(model,
57
- task="embedding",
57
+ task="embed",
58
dtype=dtype,
59
max_model_len=None,
60
**vllm_extra_kwargs) as vllm_model:
tests/models/embedding/language/test_scoring.py
+3
-9
@@ -35,9 +35,7 @@ def test_llm_1_to_1(vllm_runner, hf_runner, model_name, dtype: str):
35
with hf_runner(model_name, dtype=dtype, is_cross_encoder=True) as hf_model:
36
hf_outputs = hf_model.predict([text_pair]).tolist()
37
38
- with vllm_runner(model_name,
39
- task="embedding",
40
- dtype=dtype,
38
+ with vllm_runner(model_name, task="score", dtype=dtype,
39
max_model_len=None) as vllm_model:
40
vllm_outputs = vllm_model.score(text_pair[0], text_pair[1])
41
@@ -58,9 +56,7 @@ def test_llm_1_to_N(vllm_runner, hf_runner, model_name, dtype: str):
56
with hf_runner(model_name, dtype=dtype, is_cross_encoder=True) as hf_model:
57
hf_outputs = hf_model.predict(text_pairs).tolist()
58
61
- with vllm_runner(model_name,
62
- task="embedding",
63
- dtype=dtype,
59
+ with vllm_runner(model_name, task="score", dtype=dtype,
60
max_model_len=None) as vllm_model:
61
vllm_outputs = vllm_model.score(TEXTS_1[0], TEXTS_2)
62
@@ -82,9 +78,7 @@ def test_llm_N_to_N(vllm_runner, hf_runner, model_name, dtype: str):
78
with hf_runner(model_name, dtype=dtype, is_cross_encoder=True) as hf_model:
79
hf_outputs = hf_model.predict(text_pairs).tolist()
80
85
- with vllm_runner(model_name,
86
- task="embedding",
87
- dtype=dtype,
81
+ with vllm_runner(model_name, task="score", dtype=dtype,
82
max_model_len=None) as vllm_model:
83
vllm_outputs = vllm_model.score(TEXTS_1, TEXTS_2)
84
tests/models/embedding/vision_language/test_dse_qwen2_vl.py
+1
-1
@@ -93,7 +93,7 @@ def _run_test(
93
# if we run HF first, the cuda initialization will be done and it
94
# will hurt multiprocessing backend with fork method (the default method).
95
with vllm_runner(model,
96
- task="embedding",
96
+ task="embed",
97
dtype=dtype,
98
enforce_eager=True,
99
max_model_len=8192) as vllm_model:
tests/models/embedding/vision_language/test_llava_next.py
+1
-1
@@ -47,7 +47,7 @@ def _run_test(
47
# if we run HF first, the cuda initialization will be done and it
48
# will hurt multiprocessing backend with fork method (the default method).
49
with vllm_runner(model,
50
- task="embedding",
50
+ task="embed",
51
dtype=dtype,
52
max_model_len=4096,
53
enforce_eager=True) as vllm_model:
tests/models/embedding/vision_language/test_phi3v.py
+1
-1
@@ -39,7 +39,7 @@ def _run_test(
39
# vLLM needs a fresh new process without cuda initialization.
40
# if we run HF first, the cuda initialization will be done and it
41
# will hurt multiprocessing backend with fork method (the default method).
42
- with vllm_runner(model, task="embedding", dtype=dtype,
42
+ with vllm_runner(model, task="embed", dtype=dtype,
43
enforce_eager=True) as vllm_model:
44
vllm_outputs = vllm_model.encode(input_texts, images=input_images)
45
tests/test_config.py
+12
-5
@@ -7,11 +7,17 @@ from vllm.model_executor.layers.pooler import PoolingType
7
from vllm.platforms import current_platform
8
9
10
-@pytest.mark.parametrize(("model_id", "expected_task"), [
11
- ("facebook/opt-125m", "generate"),
12
- ("intfloat/e5-mistral-7b-instruct", "embedding"),
13
-])
14
-def test_auto_task(model_id, expected_task):
10
+@pytest.mark.parametrize(
11
+ ("model_id", "expected_runner_type", "expected_task"),
12
+ [
13
+ ("facebook/opt-125m", "generate", "generate"),
14
+ ("intfloat/e5-mistral-7b-instruct", "pooling", "embed"),
15
+ ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
16
+ ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "score"),
17
+ ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "reward"),
18
+ ],
19
+)
20
+def test_auto_task(model_id, expected_runner_type, expected_task):
21
config = ModelConfig(
22
model_id,
23
task="auto",
@@ -22,6 +28,7 @@ def test_auto_task(model_id, expected_task):
28
dtype="float16",
29
)
30
31
+ assert config.runner_type == expected_runner_type
32
assert config.task == expected_task
33
34
vllm/config.py
+95
-42
@@ -45,13 +45,27 @@ else:
45
46
logger = init_logger(__name__)
47
48
-_EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
48
+_POOLING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
49
_MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 5120
50
51
-TaskOption = Literal["auto", "generate", "embedding"]
51
+TaskOption = Literal["auto", "generate", "embedding", "embed", "classify",
52
+ "score", "reward"]
53
53
-# "draft" is only used internally for speculative decoding
54
-_Task = Literal["generate", "embedding", "draft"]
54
+_ResolvedTask = Literal["generate", "embed", "classify", "score", "reward",
55
+ "draft"]
56
+
57
+RunnerType = Literal["generate", "pooling", "draft"]
58
+
59
+_RUNNER_TASKS: Dict[RunnerType, List[_ResolvedTask]] = {
60
+ "generate": ["generate"],
61
+ "pooling": ["embed", "classify", "score", "reward"],
62
+ "draft": ["draft"],
63
+}
64
+
65
+_TASK_RUNNER: Dict[_ResolvedTask, RunnerType] = {
66
+ task: runner
67
+ for runner, tasks in _RUNNER_TASKS.items() for task in tasks
68
+}
69
70
HfOverrides = Union[Dict[str, Any], Callable[[PretrainedConfig],
71
PretrainedConfig]]
@@ -144,7 +158,7 @@ class ModelConfig:
158
def __init__(
159
self,
160
model: str,
147
- task: Union[TaskOption, _Task],
161
+ task: Union[TaskOption, Literal["draft"]],
162
tokenizer: str,
163
tokenizer_mode: str,
164
trust_remote_code: bool,
@@ -295,6 +309,7 @@ class ModelConfig:
309
supported_tasks, task = self._resolve_task(task, self.hf_config)
310
self.supported_tasks = supported_tasks
311
self.task: Final = task
312
+
313
self.pooler_config = self._init_pooler_config(override_pooler_config)
314
315
self._verify_quantization()
@@ -323,7 +338,7 @@ class ModelConfig:
338
override_pooler_config: Optional["PoolerConfig"],
339
) -> Optional["PoolerConfig"]:
340
326
- if self.task == "embedding":
341
+ if self.runner_type == "pooling":
342
user_config = override_pooler_config or PoolerConfig()
343
344
base_config = get_pooling_config(self.model, self.revision)
@@ -357,60 +372,90 @@ class ModelConfig:
372
"either 'auto', 'slow' or 'mistral'.")
373
self.tokenizer_mode = tokenizer_mode
374
375
+ def _get_preferred_task(
376
+ self,
377
+ architectures: List[str],
378
+ supported_tasks: Set[_ResolvedTask],
379
+ ) -> Optional[_ResolvedTask]:
380
+ model_id = self.model
381
+ if get_pooling_config(model_id, self.revision):
382
+ return "embed"
383
+ if ModelRegistry.is_cross_encoder_model(architectures):
384
+ return "score"
385
+
386
+ suffix_to_preferred_task: List[Tuple[str, _ResolvedTask]] = [
387
+ # Other models follow this pattern
388
+ ("ForCausalLM", "generate"),
389
+ ("ForConditionalGeneration", "generate"),
390
+ ("ForSequenceClassification", "classify"),
391
+ ("ChatModel", "generate"),
392
+ ("LMHeadModel", "generate"),
393
+ ("EmbeddingModel", "embed"),
394
+ ("RewardModel", "reward"),
395
+ ]
396
+ _, arch = ModelRegistry.inspect_model_cls(architectures)
397
+
398
+ for suffix, pref_task in suffix_to_preferred_task:
399
+ if arch.endswith(suffix) and pref_task in supported_tasks:
400
+ return pref_task
401
+
402
+ return None
403
+
404
def _resolve_task(
405
self,
362
- task_option: Union[TaskOption, _Task],
406
+ task_option: Union[TaskOption, Literal["draft"]],
407
hf_config: PretrainedConfig,
364
- ) -> Tuple[Set[_Task], _Task]:
408
+ ) -> Tuple[Set[_ResolvedTask], _ResolvedTask]:
409
if task_option == "draft":
410
return {"draft"}, "draft"
411
412
architectures = getattr(hf_config, "architectures", [])
413
370
- task_support: Dict[_Task, bool] = {
414
+ runner_support: Dict[RunnerType, bool] = {
415
# NOTE: Listed from highest to lowest priority,
416
# in case the model supports multiple of them
417
"generate": ModelRegistry.is_text_generation_model(architectures),
374
- "embedding": ModelRegistry.is_pooling_model(architectures),
418
+ "pooling": ModelRegistry.is_pooling_model(architectures),
419
}
376
- supported_tasks_lst: List[_Task] = [
377
- task for task, is_supported in task_support.items() if is_supported
420
+ supported_runner_types_lst: List[RunnerType] = [
421
+ runner_type
422
+ for runner_type, is_supported in runner_support.items()
423
+ if is_supported
424
+ ]
425
+
426
+ supported_tasks_lst: List[_ResolvedTask] = [
427
+ task for runner_type in supported_runner_types_lst
428
+ for task in _RUNNER_TASKS[runner_type]
429
]
430
supported_tasks = set(supported_tasks_lst)
431
432
if task_option == "auto":
433
selected_task = next(iter(supported_tasks_lst))
434
384
- if len(supported_tasks) > 1:
385
- suffix_to_preferred_task: List[Tuple[str, _Task]] = [
386
- # Hardcode the models that are exceptions
387
- ("AquilaModel", "generate"),
388
- ("ChatGLMModel", "generate"),
389
- # Other models follow this pattern
390
- ("ForCausalLM", "generate"),
391
- ("ForConditionalGeneration", "generate"),
392
- ("ChatModel", "generate"),
393
- ("LMHeadModel", "generate"),
394
- ("EmbeddingModel", "embedding"),
395
- ("RewardModel", "embedding"),
396
- ("ForSequenceClassification", "embedding"),
397
- ]
398
- info, arch = ModelRegistry.inspect_model_cls(architectures)
399
-
400
- for suffix, pref_task in suffix_to_preferred_task:
401
- if arch.endswith(suffix) and pref_task in supported_tasks:
402
- selected_task = pref_task
403
- break
404
- else:
405
- if (arch.endswith("Model")
406
- and info.architecture.endswith("ForCausalLM")
407
- and "embedding" in supported_tasks):
408
- selected_task = "embedding"
435
+ if len(supported_tasks_lst) > 1:
436
+ preferred_task = self._get_preferred_task(
437
+ architectures, supported_tasks)
438
+ if preferred_task is not None:
439
+ selected_task = preferred_task
440
441
logger.info(
442
"This model supports multiple tasks: %s. "
443
"Defaulting to '%s'.", supported_tasks, selected_task)
444
else:
445
+ # Aliases
446
+ if task_option == "embedding":
447
+ preferred_task = self._get_preferred_task(
448
+ architectures, supported_tasks)
449
+ if preferred_task != "embed":
450
+ msg = ("The 'embedding' task will be restricted to "
451
+ "embedding models in a future release. Please "
452
+ "pass `--task classify`, `--task score`, or "
453
+ "`--task reward` explicitly for other pooling "
454
+ "models.")
455
+ warnings.warn(msg, DeprecationWarning, stacklevel=2)
456
+
457
+ task_option = preferred_task or "embed"
458
+
459
if task_option not in supported_tasks:
460
msg = (
461
f"This model does not support the '{task_option}' task. "
@@ -533,7 +578,7 @@ class ModelConfig:
578
579
# Async postprocessor is not necessary with embedding mode
580
# since there is no token generation
536
- if self.task == "embedding":
581
+ if self.runner_type == "pooling":
582
self.use_async_output_proc = False
583
584
# Reminder: Please update docs/source/usage/compatibility_matrix.rst
@@ -750,6 +795,14 @@ class ModelConfig:
795
architectures = getattr(self.hf_config, "architectures", [])
796
return ModelRegistry.is_cross_encoder_model(architectures)
797
798
+ @property
799
+ def supported_runner_types(self) -> Set[RunnerType]:
800
+ return {_TASK_RUNNER[task] for task in self.supported_tasks}
801
+
802
+ @property
803
+ def runner_type(self) -> RunnerType:
804
+ return _TASK_RUNNER[self.task]
805
+
806
807
class CacheConfig:
808
"""Configuration for the KV cache.
@@ -1096,7 +1149,7 @@ class ParallelConfig:
1149
class SchedulerConfig:
1150
"""Scheduler configuration."""
1151
1099
- task: str = "generate" # The task to use the model for.
1152
+ runner_type: str = "generate" # The runner type to launch for the model.
1153
1154
# Maximum number of tokens to be processed in a single iteration.
1155
max_num_batched_tokens: int = field(default=None) # type: ignore
@@ -1164,11 +1217,11 @@ class SchedulerConfig:
1217
# for higher throughput.
1218
self.max_num_batched_tokens = max(self.max_model_len, 2048)
1219
1167
- if self.task == "embedding":
1168
- # For embedding, choose specific value for higher throughput
1220
+ if self.runner_type == "pooling":
1221
+ # Choose specific value for higher throughput
1222
self.max_num_batched_tokens = max(
1223
self.max_num_batched_tokens,
1171
- _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS,
1224
+ _POOLING_MODEL_MAX_NUM_BATCHED_TOKENS,
1225
)
1226
if self.is_multimodal_model:
1227
# The value needs to be at least the number of multimodal tokens
vllm/core/scheduler.py
+1
-1
@@ -337,7 +337,7 @@ class Scheduler:
337
self.lora_config = lora_config
338
339
version = "selfattn"
340
- if (self.scheduler_config.task == "embedding"
340
+ if (self.scheduler_config.runner_type == "pooling"
341
or self.cache_config.is_attention_free):
342
version = "placeholder"
343
vllm/engine/arg_utils.py
+4
-3
@@ -1066,7 +1066,7 @@ class EngineArgs:
1066
if (is_gpu and not use_sliding_window and not use_spec_decode
1067
and not self.enable_lora
1068
and not self.enable_prompt_adapter
1069
- and model_config.task != "embedding"):
1069
+ and model_config.runner_type != "pooling"):
1070
self.enable_chunked_prefill = True
1071
logger.warning(
1072
"Chunked prefill is enabled by default for models with "
@@ -1083,7 +1083,8 @@ class EngineArgs:
1083
"errors during the initial memory profiling phase, or result "
1084
"in low performance due to small KV cache space. Consider "
1085
"setting --max-model-len to a smaller value.", max_model_len)
1086
- elif self.enable_chunked_prefill and model_config.task == "embedding":
1086
+ elif (self.enable_chunked_prefill
1087
+ and model_config.runner_type == "pooling"):
1088
msg = "Chunked prefill is not supported for embedding models"
1089
raise ValueError(msg)
1090
@@ -1144,7 +1145,7 @@ class EngineArgs:
1145
" please file an issue with detailed information.")
1146
1147
scheduler_config = SchedulerConfig(
1147
- task=model_config.task,
1148
+ runner_type=model_config.runner_type,
1149
max_num_batched_tokens=self.max_num_batched_tokens,
1150
max_num_seqs=self.max_num_seqs,
1151
max_model_len=model_config.max_model_len,
vllm/engine/llm_engine.py
+2
-2
@@ -288,7 +288,7 @@ class LLMEngine:
288
289
self.model_executor = executor_class(vllm_config=vllm_config, )
290
291
- if self.model_config.task != "embedding":
291
+ if self.model_config.runner_type != "pooling":
292
self._initialize_kv_caches()
293
294
# If usage stat is enabled, collect relevant info.
@@ -1123,7 +1123,7 @@ class LLMEngine:
1123
seq_group.metrics.model_execute_time = (
1124
o.model_execute_time)
1125
1126
- if self.model_config.task == "embedding":
1126
+ if self.model_config.runner_type == "pooling":
1127
self._process_sequence_group_outputs(seq_group, output)
1128
else:
1129
self.output_processor.process_prompt_logprob(seq_group, output)
vllm/entrypoints/llm.py
+29
-24
@@ -381,19 +381,20 @@ class LLM:
381
considered legacy and may be deprecated in the future. You should
382
instead pass them via the ``inputs`` parameter.
383
"""
384
- task = self.llm_engine.model_config.task
385
- if task != "generate":
384
+ runner_type = self.llm_engine.model_config.runner_type
385
+ if runner_type != "generate":
386
messages = [
387
"LLM.generate() is only supported for (conditional) generation "
388
"models (XForCausalLM, XForConditionalGeneration).",
389
]
390
391
- supported_tasks = self.llm_engine.model_config.supported_tasks
392
- if "generate" in supported_tasks:
391
+ supported_runner_types = self.llm_engine.model_config \
392
+ .supported_runner_types
393
+ if "generate" in supported_runner_types:
394
messages.append(
394
- "Your model supports the 'generate' task, but is "
395
- f"currently initialized for the '{task}' task. Please "
396
- "initialize the model using `--task generate`.")
395
+ "Your model supports the 'generate' runner, but is "
396
+ f"currently initialized for the '{runner_type}' runner. "
397
+ "Please initialize vLLM using `--task generate`.")
398
399
raise ValueError(" ".join(messages))
400
@@ -793,16 +794,18 @@ class LLM:
794
considered legacy and may be deprecated in the future. You should
795
instead pass them via the ``inputs`` parameter.
796
"""
796
- task = self.llm_engine.model_config.task
797
- if task != "embedding":
798
- messages = ["LLM.encode() is only supported for embedding models."]
797
+ runner_type = self.llm_engine.model_config.runner_type
798
+ if runner_type != "pooling":
799
+ messages = ["LLM.encode() is only supported for pooling models."]
800
800
- supported_tasks = self.llm_engine.model_config.supported_tasks
801
- if "embedding" in supported_tasks:
801
+ supported_runner_types = self.llm_engine.model_config \
802
+ .supported_runner_types
803
+ if "pooling" in supported_runner_types:
804
messages.append(
803
- "Your model supports the 'embedding' task, but is "
804
- f"currently initialized for the '{task}' task. Please "
805
- "initialize the model using `--task embedding`.")
805
+ "Your model supports the 'pooling' runner, but is "
806
+ f"currently initialized for the '{runner_type}' runner. "
807
+ "Please initialize vLLM using `--task embed`, "
808
+ "`--task classify`, `--task score` etc.")
809
810
raise ValueError(" ".join(messages))
811
@@ -864,21 +867,23 @@ class LLM:
867
A list of ``PoolingRequestOutput`` objects containing the
868
generated scores in the same order as the input prompts.
869
"""
867
- task = self.llm_engine.model_config.task
868
- if task != "embedding":
869
- messages = ["LLM.score() is only supported for embedding models."]
870
+ runner_type = self.llm_engine.model_config.runner_type
871
+ if runner_type != "pooling":
872
+ messages = ["LLM.score() is only supported for pooling models."]
873
871
- supported_tasks = self.llm_engine.model_config.supported_tasks
872
- if "embedding" in supported_tasks:
874
+ supported_runner_types = self.llm_engine.model_config \
875
+ .supported_runner_types
876
+ if "pooling" in supported_runner_types:
877
messages.append(
874
- "Your model supports the 'embedding' task, but is "
875
- f"currently initialized for the '{task}' task. Please "
876
- "initialize the model using `--task embedding`.")
878
+ "Your model supports the 'pooling' runner, but is "
879
+ f"currently initialized for the '{runner_type}' runner. "
880
+ "Please initialize vLLM using `--task embed`, "
881
+ "`--task classify`, `--task score` etc.")
882
883
raise ValueError(" ".join(messages))
884
885
if not self.llm_engine.model_config.is_cross_encoder:
881
- raise ValueError("Your model does not support the cross encoding")
886
+ raise ValueError("Your model does not support cross encoding")
887
888
tokenizer = self.llm_engine.get_tokenizer()
889
vllm/entrypoints/openai/api_server.py
+4
-4
@@ -573,7 +573,7 @@ def init_app_state(
573
enable_auto_tools=args.enable_auto_tool_choice,
574
tool_parser=args.tool_call_parser,
575
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
576
- ) if model_config.task == "generate" else None
576
+ ) if model_config.runner_type == "generate" else None
577
state.openai_serving_completion = OpenAIServingCompletion(
578
engine_client,
579
model_config,
@@ -582,7 +582,7 @@ def init_app_state(
582
prompt_adapters=args.prompt_adapters,
583
request_logger=request_logger,
584
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
585
- ) if model_config.task == "generate" else None
585
+ ) if model_config.runner_type == "generate" else None
586
state.openai_serving_embedding = OpenAIServingEmbedding(
587
engine_client,
588
model_config,
@@ -590,13 +590,13 @@ def init_app_state(
590
request_logger=request_logger,
591
chat_template=resolved_chat_template,
592
chat_template_content_format=args.chat_template_content_format,
593
- ) if model_config.task == "embedding" else None
593
+ ) if model_config.runner_type == "pooling" else None
594
state.openai_serving_scores = OpenAIServingScores(
595
engine_client,
596
model_config,
597
base_model_paths,
598
request_logger=request_logger
599
- ) if (model_config.task == "embedding" \
599
+ ) if (model_config.runner_type == "pooling" \
600
and model_config.is_cross_encoder) else None
601
state.openai_serving_tokenization = OpenAIServingTokenization(
602
engine_client,
vllm/entrypoints/openai/run_batch.py
+2
-2
@@ -224,7 +224,7 @@ async def main(args):
224
chat_template=None,
225
chat_template_content_format="auto",
226
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
227
- ) if model_config.task == "generate" else None
227
+ ) if model_config.runner_type == "generate" else None
228
openai_serving_embedding = OpenAIServingEmbedding(
229
engine,
230
model_config,
@@ -232,7 +232,7 @@ async def main(args):
232
request_logger=request_logger,
233
chat_template=None,
234
chat_template_content_format="auto",
235
- ) if model_config.task == "embedding" else None
235
+ ) if model_config.runner_type == "pooling" else None
236
237
tracker = BatchProgressTracker()
238
logger.info("Reading batch from %s...", args.input_file)
vllm/model_executor/model_loader/utils.py
+1
-1
@@ -35,7 +35,7 @@ def get_model_architecture(
35
architectures = ["QuantMixtralForCausalLM"]
36
37
model_cls, arch = ModelRegistry.resolve_model_cls(architectures)
38
- if model_config.task == "embedding":
38
+ if model_config.runner_type == "pooling":
39
model_cls = as_embedding_model(model_cls)
40
41
return model_cls, arch
vllm/v1/engine/core.py
+1
-1
@@ -42,7 +42,7 @@ class EngineCore:
42
executor_class: Type[Executor],
43
usage_context: UsageContext,
44
):
45
- assert vllm_config.model_config.task != "embedding"
45
+ assert vllm_config.model_config.runner_type != "pooling"
46
47
logger.info("Initializing an LLM engine (v%s) with config: %s",
48
VLLM_VERSION, vllm_config)
vllm/worker/cpu_worker.py
+1
-1
@@ -163,7 +163,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
163
not in ["medusa", "mlp_speculator", "eagle"]) \
164
else {"return_hidden_states": True}
165
ModelRunnerClass: Type[CPUModelRunnerBase] = CPUModelRunner
166
- if self.model_config.task == "embedding":
166
+ if self.model_config.runner_type == "pooling":
167
ModelRunnerClass = CPUPoolingModelRunner
168
elif self.model_config.is_encoder_decoder:
169
ModelRunnerClass = CPUEncoderDecoderModelRunner
vllm/worker/worker.py
+1
-1
@@ -75,7 +75,7 @@ class Worker(LocalOrDistributedWorkerBase):
75
else {"return_hidden_states": True}
76
77
ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
78
- if model_config.task == "embedding":
78
+ if model_config.runner_type == "pooling":
79
ModelRunnerClass = PoolingModelRunner
80
elif self.model_config.is_encoder_decoder:
81
ModelRunnerClass = EncoderDecoderModelRunner