@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f10d5e393
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Split up pooling tasks (#10820)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 11, 2024 at 17:28 UTC 8f10d5e3930f05c2057a831cd80ba24c52b8ceef
27 files changed +527 -168
docs/source/index.rst
+2
@@ -94,6 +94,8 @@ Documentation
94 :caption: Models
95
96 models/supported_models
97 + models/generative_models
98 + models/pooling_models
99 models/adding_model
100 models/enabling_multimodal_inputs
101
docs/source/models/generative_models.rst new
+146
@@ -0,0 +1,146 @@
1 +.. _generative_models:
2 +
3 +Generative Models
4 +=================
5 +
6 +vLLM provides first-class support for generative models, which covers most of LLMs.
7 +
8 +In vLLM, generative models implement the :class:`~vllm.model_executor.models.VllmModelForTextGeneration` interface.
9 +Based on the final hidden states of the input, these models output log probabilities of the tokens to generate,
10 +which are then passed through :class:`~vllm.model_executor.layers.Sampler` to obtain the final text.
11 +
12 +Offline Inference
13 +-----------------
14 +
15 +The :class:`~vllm.LLM` class provides various methods for offline inference.
16 +See :ref:`Engine Arguments <engine_args>` for a list of options when initializing the model.
17 +
18 +For generative models, the only supported :code:`task` option is :code:`"generate"`.
19 +Usually, this is automatically inferred so you don't have to specify it.
20 +
21 +``LLM.generate``
22 +^^^^^^^^^^^^^^^^
23 +
24 +The :class:`~vllm.LLM.generate` method is available to all generative models in vLLM.
25 +It is similar to `its counterpart in HF Transformers <https://huggingface.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate>`__,
26 +except that tokenization and detokenization are also performed automatically.
27 +
28 +.. code-block:: python
29 +
30 + llm = LLM(model="facebook/opt-125m")
31 + outputs = llm.generate("Hello, my name is")
32 +
33 + for output in outputs:
34 + prompt = output.prompt
35 + generated_text = output.outputs[0].text
36 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
37 +
38 +You can optionally control the language generation by passing :class:`~vllm.SamplingParams`.
39 +For example, you can use greedy sampling by setting :code:`temperature=0`:
40 +
41 +.. code-block:: python
42 +
43 + llm = LLM(model="facebook/opt-125m")
44 + params = SamplingParams(temperature=0)
45 + outputs = llm.generate("Hello, my name is", params)
46 +
47 + for output in outputs:
48 + prompt = output.prompt
49 + generated_text = output.outputs[0].text
50 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
51 +
52 +A code example can be found in `examples/offline_inference.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference.py>`_.
53 +
54 +``LLM.beam_search``
55 +^^^^^^^^^^^^^^^^^^^
56 +
57 +The :class:`~vllm.LLM.beam_search` method implements `beam search <https://huggingface.co/docs/transformers/en/generation_strategies#beam-search-decoding>`__ on top of :class:`~vllm.LLM.generate`.
58 +For example, to search using 5 beams and output at most 50 tokens:
59 +
60 +.. code-block:: python
61 +
62 + llm = LLM(model="facebook/opt-125m")
63 + params = BeamSearchParams(beam_width=5, max_tokens=50)
64 + outputs = llm.generate("Hello, my name is", params)
65 +
66 + for output in outputs:
67 + prompt = output.prompt
68 + generated_text = output.outputs[0].text
69 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
70 +
71 +``LLM.chat``
72 +^^^^^^^^^^^^
73 +
74 +The :class:`~vllm.LLM.chat` method implements chat functionality on top of :class:`~vllm.LLM.generate`.
75 +In particular, it accepts input similar to `OpenAI Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`__
76 +and automatically applies the model's `chat template <https://huggingface.co/docs/transformers/en/chat_templating>`__ to format the prompt.
77 +
78 +.. important::
79 +
80 + In general, only instruction-tuned models have a chat template.
81 + Base models may perform poorly as they are not trained to respond to the chat conversation.
82 +
83 +.. code-block:: python
84 +
85 + llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
86 + conversation = [
87 + {
88 + "role": "system",
89 + "content": "You are a helpful assistant"
90 + },
91 + {
92 + "role": "user",
93 + "content": "Hello"
94 + },
95 + {
96 + "role": "assistant",
97 + "content": "Hello! How can I assist you today?"
98 + },
99 + {
100 + "role": "user",
101 + "content": "Write an essay about the importance of higher education.",
102 + },
103 + ]
104 + outputs = llm.chat(conversation)
105 +
106 + for output in outputs:
107 + prompt = output.prompt
108 + generated_text = output.outputs[0].text
109 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
110 +
111 +A code example can be found in `examples/offline_inference_chat.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_chat.py>`_.
112 +
113 +If the model doesn't have a chat template or you want to specify another one,
114 +you can explicitly pass a chat template:
115 +
116 +.. code-block:: python
117 +
118 + from vllm.entrypoints.chat_utils import load_chat_template
119 +
120 + # You can find a list of existing chat templates under `examples/`
121 + custom_template = load_chat_template(chat_template="<path_to_template>")
122 + print("Loaded chat template:", custom_template)
123 +
124 + outputs = llm.chat(conversation, chat_template=custom_template)
125 +
126 +Online Inference
127 +----------------
128 +
129 +Our `OpenAI Compatible Server <../serving/openai_compatible_server>`__ can be used for online inference.
130 +Please click on the above link for more details on how to launch the server.
131 +
132 +Completions API
133 +^^^^^^^^^^^^^^^
134 +
135 +Our Completions API is similar to ``LLM.generate`` but only accepts text.
136 +It is compatible with `OpenAI Completions API <https://platform.openai.com/docs/api-reference/completions>`__
137 +so that you can use OpenAI client to interact with it.
138 +A code example can be found in `examples/openai_completion_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_completion_client.py>`_.
139 +
140 +Chat API
141 +^^^^^^^^
142 +
143 +Our Chat API is similar to ``LLM.chat``, accepting both text and :ref:`multi-modal inputs <multimodal_inputs>`.
144 +It is compatible with `OpenAI Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`__
145 +so that you can use OpenAI client to interact with it.
146 +A code example can be found in `examples/openai_chat_completion_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_client.py>`_.
docs/source/models/pooling_models.rst new
+99
@@ -0,0 +1,99 @@
1 +.. _pooling_models:
2 +
3 +Pooling Models
4 +==============
5 +
6 +vLLM also supports pooling models, including embedding, reranking and reward models.
7 +
8 +In vLLM, pooling models implement the :class:`~vllm.model_executor.models.VllmModelForPooling` interface.
9 +These models use a :class:`~vllm.model_executor.layers.Pooler` to aggregate the final hidden states of the input
10 +before returning them.
11 +
12 +.. note::
13 +
14 + We currently support pooling models primarily as a matter of convenience.
15 + As shown in the :ref:`Compatibility Matrix <compatibility_matrix>`, most vLLM features are not applicable to
16 + pooling models as they only work on the generation or decode stage, so performance may not improve as much.
17 +
18 +Offline Inference
19 +-----------------
20 +
21 +The :class:`~vllm.LLM` class provides various methods for offline inference.
22 +See :ref:`Engine Arguments <engine_args>` for a list of options when initializing the model.
23 +
24 +For pooling models, we support the following :code:`task` options:
25 +
26 +- Embedding (:code:`"embed"` / :code:`"embedding"`)
27 +- Classification (:code:`"classify"`)
28 +- Sentence Pair Scoring (:code:`"score"`)
29 +- Reward Modeling (:code:`"reward"`)
30 +
31 +The selected task determines the default :class:`~vllm.model_executor.layers.Pooler` that is used:
32 +
33 +- Embedding: Extract only the hidden states corresponding to the last token, and apply normalization.
34 +- Classification: Extract only the hidden states corresponding to the last token, and apply softmax.
35 +- Sentence Pair Scoring: Extract only the hidden states corresponding to the last token, and apply softmax.
36 +- Reward Modeling: Extract all of the hidden states and return them directly.
37 +
38 +When loading `Sentence Transformers <https://huggingface.co/sentence-transformers>`__ models,
39 +we attempt to override the default pooler based on its Sentence Transformers configuration file (:code:`modules.json`).
40 +
41 +You can customize the model's pooling method via the :code:`override_pooler_config` option,
42 +which takes priority over both the model's and Sentence Transformers's defaults.
43 +
44 +``LLM.encode``
45 +^^^^^^^^^^^^^^
46 +
47 +The :class:`~vllm.LLM.encode` method is available to all pooling models in vLLM.
48 +It returns the aggregated hidden states directly.
49 +
50 +.. code-block:: python
51 +
52 + llm = LLM(model="intfloat/e5-mistral-7b-instruct", task="embed")
53 + outputs = llm.encode("Hello, my name is")
54 +
55 + outputs = model.encode(prompts)
56 + for output in outputs:
57 + embeddings = output.outputs.embedding
58 + print(f"Prompt: {prompt!r}, Embeddings (size={len(embeddings)}: {embeddings!r}")
59 +
60 +A code example can be found in `examples/offline_inference_embedding.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_embedding.py>`_.
61 +
62 +``LLM.score``
63 +^^^^^^^^^^^^^
64 +
65 +The :class:`~vllm.LLM.score` method outputs similarity scores between sentence pairs.
66 +It is primarily designed for `cross-encoder models <https://www.sbert.net/examples/applications/cross-encoder/README.html>`__.
67 +These types of models serve as rerankers between candidate query-document pairs in RAG systems.
68 +
69 +.. note::
70 +
71 + vLLM can only perform the model inference component (e.g. embedding, reranking) of RAG.
72 + To handle RAG at a higher level, you should use integration frameworks such as `LangChain <https://github.com/langchain-ai/langchain>`_.
73 +
74 +You can use `these tests <https://github.com/vllm-project/vllm/blob/main/tests/models/embedding/language/test_scoring.py>`_ as reference.
75 +
76 +Online Inference
77 +----------------
78 +
79 +Our `OpenAI Compatible Server <../serving/openai_compatible_server>`__ can be used for online inference.
80 +Please click on the above link for more details on how to launch the server.
81 +
82 +Embeddings API
83 +^^^^^^^^^^^^^^
84 +
85 +Our Embeddings API is similar to ``LLM.encode``, accepting both text and :ref:`multi-modal inputs <multimodal_inputs>`.
86 +
87 +The text-only API is compatible with `OpenAI Embeddings API <https://platform.openai.com/docs/api-reference/embeddings>`__
88 +so that you can use OpenAI client to interact with it.
89 +A code example can be found in `examples/openai_embedding_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_embedding_client.py>`_.
90 +
91 +The multi-modal API is an extension of the `OpenAI Embeddings API <https://platform.openai.com/docs/api-reference/embeddings>`__
92 +that incorporates `OpenAI Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`__,
93 +so it is not part of the OpenAI standard. Please see :ref:`this page <multimodal_inputs>` for more details on how to use it.
94 +
95 +Score API
96 +^^^^^^^^^
97 +
98 +Our Score API is similar to ``LLM.score``.
99 +Please see `this page <../serving/openai_compatible_server.html#score-api-for-cross-encoder-models>`__ for more details on how to use it.
docs/source/models/supported_models.rst
+102 -55
@@ -3,11 +3,21 @@
3 Supported Models
4 ================
5
6 -vLLM supports a variety of generative and embedding models from `HuggingFace (HF) Transformers <https://huggingface.co/models>`_.
7 -This page lists the model architectures that are currently supported by vLLM.
6 +vLLM supports generative and pooling models across various tasks.
7 +If a model supports more than one task, you can set the task via the :code:`--task` argument.
8 +
9 +For each task, we list the model architectures that have been implemented in vLLM.
10 Alongside each architecture, we include some popular models that use it.
11
10 -For other models, you can check the :code:`config.json` file inside the model repository.
12 +Loading a Model
13 +^^^^^^^^^^^^^^^
14 +
15 +HuggingFace Hub
16 ++++++++++++++++
17 +
18 +By default, vLLM loads models from `HuggingFace (HF) Hub <https://huggingface.co/models>`_.
19 +
20 +To determine whether a given model is supported, you can check the :code:`config.json` file inside the HF repository.
21 If the :code:`"architectures"` field contains a model architecture listed below, then it should be supported in theory.
22
23 .. tip::
@@ -17,38 +27,57 @@ If the :code:`"architectures"` field contains a model architecture listed below,
27
28 from vllm import LLM
29
20 - llm = LLM(model=...) # Name or path of your model
30 + # For generative models (task=generate) only
31 + llm = LLM(model=..., task="generate") # Name or path of your model
32 output = llm.generate("Hello, my name is")
33 print(output)
34
24 - If vLLM successfully generates text, it indicates that your model is supported.
35 + # For pooling models (task={embed,classify,reward}) only
36 + llm = LLM(model=..., task="embed") # Name or path of your model
37 + output = llm.encode("Hello, my name is")
38 + print(output)
39 +
40 + If vLLM successfully returns text (for generative models) or hidden states (for pooling models), it indicates that your model is supported.
41
42 Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` and :ref:`Enabling Multimodal Inputs <enabling_multimodal_inputs>`
43 for instructions on how to implement your model in vLLM.
44 Alternatively, you can `open an issue on GitHub <https://github.com/vllm-project/vllm/issues/new/choose>`_ to request vLLM support.
45
30 -.. note::
31 - To use models from `ModelScope <https://www.modelscope.cn>`_ instead of HuggingFace Hub, set an environment variable:
46 +ModelScope
47 +++++++++++
48
33 - .. code-block:: shell
49 +To use models from `ModelScope <https://www.modelscope.cn>`_ instead of HuggingFace Hub, set an environment variable:
50
35 - $ export VLLM_USE_MODELSCOPE=True
51 +.. code-block:: shell
52
37 - And use with :code:`trust_remote_code=True`.
53 + $ export VLLM_USE_MODELSCOPE=True
54
39 - .. code-block:: python
55 +And use with :code:`trust_remote_code=True`.
56
41 - from vllm import LLM
57 +.. code-block:: python
58
43 - llm = LLM(model=..., revision=..., trust_remote_code=True) # Name or path of your model
44 - output = llm.generate("Hello, my name is")
45 - print(output)
59 + from vllm import LLM
60 +
61 + llm = LLM(model=..., revision=..., task=..., trust_remote_code=True)
62
47 -Text-only Language Models
48 -^^^^^^^^^^^^^^^^^^^^^^^^^
63 + # For generative models (task=generate) only
64 + output = llm.generate("Hello, my name is")
65 + print(output)
66
50 -Text Generation
51 ----------------
67 + # For pooling models (task={embed,classify,reward}) only
68 + output = llm.encode("Hello, my name is")
69 + print(output)
70 +
71 +List of Text-only Language Models
72 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
73 +
74 +Generative Models
75 ++++++++++++++++++
76 +
77 +See :ref:`this page <generative_models>` for more information on how to use generative models.
78 +
79 +Text Generation (``--task generate``)
80 +-------------------------------------
81
82 .. list-table::
83 :widths: 25 25 50 5 5
@@ -328,8 +357,24 @@ Text Generation
357 .. note::
358 Currently, the ROCm version of vLLM supports Mistral and Mixtral only for context lengths up to 4096.
359
331 -Text Embedding
332 ---------------
360 +Pooling Models
361 +++++++++++++++
362 +
363 +See :ref:`this page <pooling_models>` for more information on how to use pooling models.
364 +
365 +.. important::
366 + Since some model architectures support both generative and pooling tasks,
367 + you should explicitly specify the task type to ensure that the model is used in pooling mode instead of generative mode.
368 +
369 +Text Embedding (``--task embed``)
370 +---------------------------------
371 +
372 +Any text generation model can be converted into an embedding model by passing :code:`--task embed`.
373 +
374 +.. note::
375 + To get the best results, you should use pooling models that are specifically trained as such.
376 +
377 +The following table lists those that are tested in vLLM.
378
379 .. list-table::
380 :widths: 25 25 50 5 5
@@ -371,13 +416,6 @@ Text Embedding
416 -
417 -
418
374 -.. important::
375 - Some model architectures support both generation and embedding tasks.
376 - In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
377 -
378 -.. tip::
379 - You can override the model's pooling method by passing :code:`--override-pooler-config`.
380 -
419 .. note::
420 :code:`ssmits/Qwen2-7B-Instruct-embed-base` has an improperly defined Sentence Transformers config.
421 You should manually set mean pooling by passing :code:`--override-pooler-config '{"pooling_type": "MEAN"}'`.
@@ -389,8 +427,8 @@ Text Embedding
427 On the other hand, its 1.5B variant (:code:`Alibaba-NLP/gte-Qwen2-1.5B-instruct`) uses causal attention
428 despite being described otherwise on its model card.
429
392 -Reward Modeling
393 ----------------
430 +Reward Modeling (``--task reward``)
431 +-----------------------------------
432
433 .. list-table::
434 :widths: 25 25 50 5 5
@@ -416,11 +454,8 @@ Reward Modeling
454 For process-supervised reward models such as :code:`peiyi9979/math-shepherd-mistral-7b-prm`, the pooling config should be set explicitly,
455 e.g.: :code:`--override-pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
456
419 -.. note::
420 - As an interim measure, these models are supported in both offline and online inference via Embeddings API.
421 -
422 -Classification
423 ----------------
457 +Classification (``--task classify``)
458 +------------------------------------
459
460 .. list-table::
461 :widths: 25 25 50 5 5
@@ -437,11 +472,8 @@ Classification
472 - ✅︎
473 - ✅︎
474
440 -.. note::
441 - As an interim measure, these models are supported in both offline and online inference via Embeddings API.
442 -
443 -Sentence Pair Scoring
444 ----------------------
475 +Sentence Pair Scoring (``--task score``)
476 +----------------------------------------
477
478 .. list-table::
479 :widths: 25 25 50 5 5
@@ -468,13 +500,10 @@ Sentence Pair Scoring
500 -
501 -
502
471 -.. note::
472 - These models are supported in both offline and online inference via Score API.
473 -
503 .. _supported_mm_models:
504
476 -Multimodal Language Models
477 -^^^^^^^^^^^^^^^^^^^^^^^^^^
505 +List of Multimodal Language Models
506 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
507
508 The following modalities are supported depending on the model:
509
@@ -491,8 +520,15 @@ On the other hand, modalities separated by :code:`/` are mutually exclusive.
520
521 - e.g.: :code:`T / I` means that the model supports text-only and image-only inputs, but not text-with-image inputs.
522
494 -Text Generation
495 ----------------
523 +See :ref:`this page <multimodal_inputs>` on how to pass multi-modal inputs to the model.
524 +
525 +Generative Models
526 ++++++++++++++++++
527 +
528 +See :ref:`this page <generative_models>` for more information on how to use generative models.
529 +
530 +Text Generation (``--task generate``)
531 +-------------------------------------
532
533 .. list-table::
534 :widths: 25 25 15 20 5 5 5
@@ -696,8 +732,24 @@ Text Generation
732 The official :code:`openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
733 For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
734
699 -Multimodal Embedding
700 ---------------------
735 +Pooling Models
736 +++++++++++++++
737 +
738 +See :ref:`this page <pooling_models>` for more information on how to use pooling models.
739 +
740 +.. important::
741 + Since some model architectures support both generative and pooling tasks,
742 + you should explicitly specify the task type to ensure that the model is used in pooling mode instead of generative mode.
743 +
744 +Text Embedding (``--task embed``)
745 +---------------------------------
746 +
747 +Any text generation model can be converted into an embedding model by passing :code:`--task embed`.
748 +
749 +.. note::
750 + To get the best results, you should use pooling models that are specifically trained as such.
751 +
752 +The following table lists those that are tested in vLLM.
753
754 .. list-table::
755 :widths: 25 25 15 25 5 5
@@ -728,12 +780,7 @@ Multimodal Embedding
780 -
781 - ✅︎
782
731 -.. important::
732 - Some model architectures support both generation and embedding tasks.
733 - In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
734 -
735 -.. tip::
736 - You can override the model's pooling method by passing :code:`--override-pooler-config`.
783 +----
784
785 Model Support Policy
786 =====================
docs/source/usage/compatibility_matrix.rst
+6 -6
@@ -39,13 +39,13 @@ Feature x Feature
39 - :abbr:`prmpt adptr (Prompt Adapter)`
40 - :ref:`SD <spec_decode>`
41 - CUDA graph
42 - - :abbr:`emd (Embedding Models)`
42 + - :abbr:`pooling (Pooling Models)`
43 - :abbr:`enc-dec (Encoder-Decoder Models)`
44 - :abbr:`logP (Logprobs)`
45 - :abbr:`prmpt logP (Prompt Logprobs)`
46 - :abbr:`async output (Async Output Processing)`
47 - multi-step
48 - - :abbr:`mm (Multimodal)`
48 + - :abbr:`mm (Multimodal Inputs)`
49 - best-of
50 - beam-search
51 - :abbr:`guided dec (Guided Decoding)`
@@ -151,7 +151,7 @@ Feature x Feature
151 -
152 -
153 -
154 - * - :abbr:`emd (Embedding Models)`
154 + * - :abbr:`pooling (Pooling Models)`
155 - ✗
156 - ✗
157 - ✗
@@ -253,7 +253,7 @@ Feature x Feature
253 -
254 -
255 -
256 - * - :abbr:`mm (Multimodal)`
256 + * - :abbr:`mm (Multimodal Inputs)`
257 - ✅
258 - `✗ <https://github.com/vllm-project/vllm/pull/8348>`__
259 - `✗ <https://github.com/vllm-project/vllm/pull/7199>`__
@@ -386,7 +386,7 @@ Feature x Hardware
386 - ✅
387 - ✗
388 - ✅
389 - * - :abbr:`emd (Embedding Models)`
389 + * - :abbr:`pooling (Pooling Models)`
390 - ✅
391 - ✅
392 - ✅
@@ -402,7 +402,7 @@ Feature x Hardware
402 - ✅
403 - ✅
404 - ✗
405 - * - :abbr:`mm (Multimodal)`
405 + * - :abbr:`mm (Multimodal Inputs)`
406 - ✅
407 - ✅
408 - ✅
examples/offline_inference_embedding.py
+6 -1
@@ -9,7 +9,12 @@ prompts = [
9 ]
10
11 # Create an LLM.
12 -model = LLM(model="intfloat/e5-mistral-7b-instruct", enforce_eager=True)
12 +model = LLM(
13 + model="intfloat/e5-mistral-7b-instruct",
14 + task="embed", # You should pass task="embed" for embedding models
15 + enforce_eager=True,
16 +)
17 +
18 # Generate embedding. The output is a list of PoolingRequestOutputs.
19 outputs = model.encode(prompts)
20 # Print the outputs.
examples/offline_inference_vision_language_embedding.py
+2 -2
@@ -59,7 +59,7 @@ def run_e5_v(query: Query):
59
60 llm = LLM(
61 model="royokong/e5-v",
62 - task="embedding",
62 + task="embed",
63 max_model_len=4096,
64 )
65
@@ -88,7 +88,7 @@ def run_vlm2vec(query: Query):
88
89 llm = LLM(
90 model="TIGER-Lab/VLM2Vec-Full",
91 - task="embedding",
91 + task="embed",
92 trust_remote_code=True,
93 mm_processor_kwargs={"num_crops": 4},
94 )
tests/compile/test_basic_correctness.py
+2 -2
@@ -55,7 +55,7 @@ test_settings = [
55 # embedding model
56 TestSetting(
57 model="BAAI/bge-multilingual-gemma2",
58 - model_args=["--task", "embedding"],
58 + model_args=["--task", "embed"],
59 pp_size=1,
60 tp_size=1,
61 attn_backend="FLASHINFER",
@@ -65,7 +65,7 @@ test_settings = [
65 # encoder-based embedding model (BERT)
66 TestSetting(
67 model="BAAI/bge-base-en-v1.5",
68 - model_args=["--task", "embedding"],
68 + model_args=["--task", "embed"],
69 pp_size=1,
70 tp_size=1,
71 attn_backend="XFORMERS",
tests/core/test_scheduler_encoder_decoder.py
+1 -1
@@ -37,7 +37,7 @@ def test_scheduler_schedule_simple_encoder_decoder():
37 num_seq_group = 4
38 max_model_len = 16
39 scheduler_config = SchedulerConfig(
40 - task="generate",
40 + "generate",
41 max_num_batched_tokens=64,
42 max_num_seqs=num_seq_group,
43 max_model_len=max_model_len,
tests/entrypoints/openai/test_vision_embedding.py
+1 -1
@@ -27,7 +27,7 @@ TEST_IMAGE_URLS = [
27 def server():
28 args = [
29 "--task",
30 - "embedding",
30 + "embed",
31 "--dtype",
32 "bfloat16",
33 "--max-model-len",
tests/models/embedding/language/test_embedding.py
+1 -1
@@ -54,7 +54,7 @@ def test_models(
54 hf_outputs = hf_model.encode(example_prompts)
55
56 with vllm_runner(model,
57 - task="embedding",
57 + task="embed",
58 dtype=dtype,
59 max_model_len=None,
60 **vllm_extra_kwargs) as vllm_model:
tests/models/embedding/language/test_scoring.py
+3 -9
@@ -35,9 +35,7 @@ def test_llm_1_to_1(vllm_runner, hf_runner, model_name, dtype: str):
35 with hf_runner(model_name, dtype=dtype, is_cross_encoder=True) as hf_model:
36 hf_outputs = hf_model.predict([text_pair]).tolist()
37
38 - with vllm_runner(model_name,
39 - task="embedding",
40 - dtype=dtype,
38 + with vllm_runner(model_name, task="score", dtype=dtype,
39 max_model_len=None) as vllm_model:
40 vllm_outputs = vllm_model.score(text_pair[0], text_pair[1])
41
@@ -58,9 +56,7 @@ def test_llm_1_to_N(vllm_runner, hf_runner, model_name, dtype: str):
56 with hf_runner(model_name, dtype=dtype, is_cross_encoder=True) as hf_model:
57 hf_outputs = hf_model.predict(text_pairs).tolist()
58
61 - with vllm_runner(model_name,
62 - task="embedding",
63 - dtype=dtype,
59 + with vllm_runner(model_name, task="score", dtype=dtype,
60 max_model_len=None) as vllm_model:
61 vllm_outputs = vllm_model.score(TEXTS_1[0], TEXTS_2)
62
@@ -82,9 +78,7 @@ def test_llm_N_to_N(vllm_runner, hf_runner, model_name, dtype: str):
78 with hf_runner(model_name, dtype=dtype, is_cross_encoder=True) as hf_model:
79 hf_outputs = hf_model.predict(text_pairs).tolist()
80
85 - with vllm_runner(model_name,
86 - task="embedding",
87 - dtype=dtype,
81 + with vllm_runner(model_name, task="score", dtype=dtype,
82 max_model_len=None) as vllm_model:
83 vllm_outputs = vllm_model.score(TEXTS_1, TEXTS_2)
84
tests/models/embedding/vision_language/test_dse_qwen2_vl.py
+1 -1
@@ -93,7 +93,7 @@ def _run_test(
93 # if we run HF first, the cuda initialization will be done and it
94 # will hurt multiprocessing backend with fork method (the default method).
95 with vllm_runner(model,
96 - task="embedding",
96 + task="embed",
97 dtype=dtype,
98 enforce_eager=True,
99 max_model_len=8192) as vllm_model:
tests/models/embedding/vision_language/test_llava_next.py
+1 -1
@@ -47,7 +47,7 @@ def _run_test(
47 # if we run HF first, the cuda initialization will be done and it
48 # will hurt multiprocessing backend with fork method (the default method).
49 with vllm_runner(model,
50 - task="embedding",
50 + task="embed",
51 dtype=dtype,
52 max_model_len=4096,
53 enforce_eager=True) as vllm_model:
tests/models/embedding/vision_language/test_phi3v.py
+1 -1
@@ -39,7 +39,7 @@ def _run_test(
39 # vLLM needs a fresh new process without cuda initialization.
40 # if we run HF first, the cuda initialization will be done and it
41 # will hurt multiprocessing backend with fork method (the default method).
42 - with vllm_runner(model, task="embedding", dtype=dtype,
42 + with vllm_runner(model, task="embed", dtype=dtype,
43 enforce_eager=True) as vllm_model:
44 vllm_outputs = vllm_model.encode(input_texts, images=input_images)
45
tests/test_config.py
+12 -5
@@ -7,11 +7,17 @@ from vllm.model_executor.layers.pooler import PoolingType
7 from vllm.platforms import current_platform
8
9
10 -@pytest.mark.parametrize(("model_id", "expected_task"), [
11 - ("facebook/opt-125m", "generate"),
12 - ("intfloat/e5-mistral-7b-instruct", "embedding"),
13 -])
14 -def test_auto_task(model_id, expected_task):
10 +@pytest.mark.parametrize(
11 + ("model_id", "expected_runner_type", "expected_task"),
12 + [
13 + ("facebook/opt-125m", "generate", "generate"),
14 + ("intfloat/e5-mistral-7b-instruct", "pooling", "embed"),
15 + ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
16 + ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "score"),
17 + ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "reward"),
18 + ],
19 +)
20 +def test_auto_task(model_id, expected_runner_type, expected_task):
21 config = ModelConfig(
22 model_id,
23 task="auto",
@@ -22,6 +28,7 @@ def test_auto_task(model_id, expected_task):
28 dtype="float16",
29 )
30
31 + assert config.runner_type == expected_runner_type
32 assert config.task == expected_task
33
34
vllm/config.py
+95 -42
@@ -45,13 +45,27 @@ else:
45
46 logger = init_logger(__name__)
47
48 -_EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
48 +_POOLING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
49 _MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 5120
50
51 -TaskOption = Literal["auto", "generate", "embedding"]
51 +TaskOption = Literal["auto", "generate", "embedding", "embed", "classify",
52 + "score", "reward"]
53
53 -# "draft" is only used internally for speculative decoding
54 -_Task = Literal["generate", "embedding", "draft"]
54 +_ResolvedTask = Literal["generate", "embed", "classify", "score", "reward",
55 + "draft"]
56 +
57 +RunnerType = Literal["generate", "pooling", "draft"]
58 +
59 +_RUNNER_TASKS: Dict[RunnerType, List[_ResolvedTask]] = {
60 + "generate": ["generate"],
61 + "pooling": ["embed", "classify", "score", "reward"],
62 + "draft": ["draft"],
63 +}
64 +
65 +_TASK_RUNNER: Dict[_ResolvedTask, RunnerType] = {
66 + task: runner
67 + for runner, tasks in _RUNNER_TASKS.items() for task in tasks
68 +}
69
70 HfOverrides = Union[Dict[str, Any], Callable[[PretrainedConfig],
71 PretrainedConfig]]
@@ -144,7 +158,7 @@ class ModelConfig:
158 def __init__(
159 self,
160 model: str,
147 - task: Union[TaskOption, _Task],
161 + task: Union[TaskOption, Literal["draft"]],
162 tokenizer: str,
163 tokenizer_mode: str,
164 trust_remote_code: bool,
@@ -295,6 +309,7 @@ class ModelConfig:
309 supported_tasks, task = self._resolve_task(task, self.hf_config)
310 self.supported_tasks = supported_tasks
311 self.task: Final = task
312 +
313 self.pooler_config = self._init_pooler_config(override_pooler_config)
314
315 self._verify_quantization()
@@ -323,7 +338,7 @@ class ModelConfig:
338 override_pooler_config: Optional["PoolerConfig"],
339 ) -> Optional["PoolerConfig"]:
340
326 - if self.task == "embedding":
341 + if self.runner_type == "pooling":
342 user_config = override_pooler_config or PoolerConfig()
343
344 base_config = get_pooling_config(self.model, self.revision)
@@ -357,60 +372,90 @@ class ModelConfig:
372 "either 'auto', 'slow' or 'mistral'.")
373 self.tokenizer_mode = tokenizer_mode
374
375 + def _get_preferred_task(
376 + self,
377 + architectures: List[str],
378 + supported_tasks: Set[_ResolvedTask],
379 + ) -> Optional[_ResolvedTask]:
380 + model_id = self.model
381 + if get_pooling_config(model_id, self.revision):
382 + return "embed"
383 + if ModelRegistry.is_cross_encoder_model(architectures):
384 + return "score"
385 +
386 + suffix_to_preferred_task: List[Tuple[str, _ResolvedTask]] = [
387 + # Other models follow this pattern
388 + ("ForCausalLM", "generate"),
389 + ("ForConditionalGeneration", "generate"),
390 + ("ForSequenceClassification", "classify"),
391 + ("ChatModel", "generate"),
392 + ("LMHeadModel", "generate"),
393 + ("EmbeddingModel", "embed"),
394 + ("RewardModel", "reward"),
395 + ]
396 + _, arch = ModelRegistry.inspect_model_cls(architectures)
397 +
398 + for suffix, pref_task in suffix_to_preferred_task:
399 + if arch.endswith(suffix) and pref_task in supported_tasks:
400 + return pref_task
401 +
402 + return None
403 +
404 def _resolve_task(
405 self,
362 - task_option: Union[TaskOption, _Task],
406 + task_option: Union[TaskOption, Literal["draft"]],
407 hf_config: PretrainedConfig,
364 - ) -> Tuple[Set[_Task], _Task]:
408 + ) -> Tuple[Set[_ResolvedTask], _ResolvedTask]:
409 if task_option == "draft":
410 return {"draft"}, "draft"
411
412 architectures = getattr(hf_config, "architectures", [])
413
370 - task_support: Dict[_Task, bool] = {
414 + runner_support: Dict[RunnerType, bool] = {
415 # NOTE: Listed from highest to lowest priority,
416 # in case the model supports multiple of them
417 "generate": ModelRegistry.is_text_generation_model(architectures),
374 - "embedding": ModelRegistry.is_pooling_model(architectures),
418 + "pooling": ModelRegistry.is_pooling_model(architectures),
419 }
376 - supported_tasks_lst: List[_Task] = [
377 - task for task, is_supported in task_support.items() if is_supported
420 + supported_runner_types_lst: List[RunnerType] = [
421 + runner_type
422 + for runner_type, is_supported in runner_support.items()
423 + if is_supported
424 + ]
425 +
426 + supported_tasks_lst: List[_ResolvedTask] = [
427 + task for runner_type in supported_runner_types_lst
428 + for task in _RUNNER_TASKS[runner_type]
429 ]
430 supported_tasks = set(supported_tasks_lst)
431
432 if task_option == "auto":
433 selected_task = next(iter(supported_tasks_lst))
434
384 - if len(supported_tasks) > 1:
385 - suffix_to_preferred_task: List[Tuple[str, _Task]] = [
386 - # Hardcode the models that are exceptions
387 - ("AquilaModel", "generate"),
388 - ("ChatGLMModel", "generate"),
389 - # Other models follow this pattern
390 - ("ForCausalLM", "generate"),
391 - ("ForConditionalGeneration", "generate"),
392 - ("ChatModel", "generate"),
393 - ("LMHeadModel", "generate"),
394 - ("EmbeddingModel", "embedding"),
395 - ("RewardModel", "embedding"),
396 - ("ForSequenceClassification", "embedding"),
397 - ]
398 - info, arch = ModelRegistry.inspect_model_cls(architectures)
399 -
400 - for suffix, pref_task in suffix_to_preferred_task:
401 - if arch.endswith(suffix) and pref_task in supported_tasks:
402 - selected_task = pref_task
403 - break
404 - else:
405 - if (arch.endswith("Model")
406 - and info.architecture.endswith("ForCausalLM")
407 - and "embedding" in supported_tasks):
408 - selected_task = "embedding"
435 + if len(supported_tasks_lst) > 1:
436 + preferred_task = self._get_preferred_task(
437 + architectures, supported_tasks)
438 + if preferred_task is not None:
439 + selected_task = preferred_task
440
441 logger.info(
442 "This model supports multiple tasks: %s. "
443 "Defaulting to '%s'.", supported_tasks, selected_task)
444 else:
445 + # Aliases
446 + if task_option == "embedding":
447 + preferred_task = self._get_preferred_task(
448 + architectures, supported_tasks)
449 + if preferred_task != "embed":
450 + msg = ("The 'embedding' task will be restricted to "
451 + "embedding models in a future release. Please "
452 + "pass `--task classify`, `--task score`, or "
453 + "`--task reward` explicitly for other pooling "
454 + "models.")
455 + warnings.warn(msg, DeprecationWarning, stacklevel=2)
456 +
457 + task_option = preferred_task or "embed"
458 +
459 if task_option not in supported_tasks:
460 msg = (
461 f"This model does not support the '{task_option}' task. "
@@ -533,7 +578,7 @@ class ModelConfig:
578
579 # Async postprocessor is not necessary with embedding mode
580 # since there is no token generation
536 - if self.task == "embedding":
581 + if self.runner_type == "pooling":
582 self.use_async_output_proc = False
583
584 # Reminder: Please update docs/source/usage/compatibility_matrix.rst
@@ -750,6 +795,14 @@ class ModelConfig:
795 architectures = getattr(self.hf_config, "architectures", [])
796 return ModelRegistry.is_cross_encoder_model(architectures)
797
798 + @property
799 + def supported_runner_types(self) -> Set[RunnerType]:
800 + return {_TASK_RUNNER[task] for task in self.supported_tasks}
801 +
802 + @property
803 + def runner_type(self) -> RunnerType:
804 + return _TASK_RUNNER[self.task]
805 +
806
807 class CacheConfig:
808 """Configuration for the KV cache.
@@ -1096,7 +1149,7 @@ class ParallelConfig:
1149 class SchedulerConfig:
1150 """Scheduler configuration."""
1151
1099 - task: str = "generate" # The task to use the model for.
1152 + runner_type: str = "generate" # The runner type to launch for the model.
1153
1154 # Maximum number of tokens to be processed in a single iteration.
1155 max_num_batched_tokens: int = field(default=None) # type: ignore
@@ -1164,11 +1217,11 @@ class SchedulerConfig:
1217 # for higher throughput.
1218 self.max_num_batched_tokens = max(self.max_model_len, 2048)
1219
1167 - if self.task == "embedding":
1168 - # For embedding, choose specific value for higher throughput
1220 + if self.runner_type == "pooling":
1221 + # Choose specific value for higher throughput
1222 self.max_num_batched_tokens = max(
1223 self.max_num_batched_tokens,
1171 - _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS,
1224 + _POOLING_MODEL_MAX_NUM_BATCHED_TOKENS,
1225 )
1226 if self.is_multimodal_model:
1227 # The value needs to be at least the number of multimodal tokens
vllm/core/scheduler.py
+1 -1
@@ -337,7 +337,7 @@ class Scheduler:
337 self.lora_config = lora_config
338
339 version = "selfattn"
340 - if (self.scheduler_config.task == "embedding"
340 + if (self.scheduler_config.runner_type == "pooling"
341 or self.cache_config.is_attention_free):
342 version = "placeholder"
343
vllm/engine/arg_utils.py
+4 -3
@@ -1066,7 +1066,7 @@ class EngineArgs:
1066 if (is_gpu and not use_sliding_window and not use_spec_decode
1067 and not self.enable_lora
1068 and not self.enable_prompt_adapter
1069 - and model_config.task != "embedding"):
1069 + and model_config.runner_type != "pooling"):
1070 self.enable_chunked_prefill = True
1071 logger.warning(
1072 "Chunked prefill is enabled by default for models with "
@@ -1083,7 +1083,8 @@ class EngineArgs:
1083 "errors during the initial memory profiling phase, or result "
1084 "in low performance due to small KV cache space. Consider "
1085 "setting --max-model-len to a smaller value.", max_model_len)
1086 - elif self.enable_chunked_prefill and model_config.task == "embedding":
1086 + elif (self.enable_chunked_prefill
1087 + and model_config.runner_type == "pooling"):
1088 msg = "Chunked prefill is not supported for embedding models"
1089 raise ValueError(msg)
1090
@@ -1144,7 +1145,7 @@ class EngineArgs:
1145 " please file an issue with detailed information.")
1146
1147 scheduler_config = SchedulerConfig(
1147 - task=model_config.task,
1148 + runner_type=model_config.runner_type,
1149 max_num_batched_tokens=self.max_num_batched_tokens,
1150 max_num_seqs=self.max_num_seqs,
1151 max_model_len=model_config.max_model_len,
vllm/engine/llm_engine.py
+2 -2
@@ -288,7 +288,7 @@ class LLMEngine:
288
289 self.model_executor = executor_class(vllm_config=vllm_config, )
290
291 - if self.model_config.task != "embedding":
291 + if self.model_config.runner_type != "pooling":
292 self._initialize_kv_caches()
293
294 # If usage stat is enabled, collect relevant info.
@@ -1123,7 +1123,7 @@ class LLMEngine:
1123 seq_group.metrics.model_execute_time = (
1124 o.model_execute_time)
1125
1126 - if self.model_config.task == "embedding":
1126 + if self.model_config.runner_type == "pooling":
1127 self._process_sequence_group_outputs(seq_group, output)
1128 else:
1129 self.output_processor.process_prompt_logprob(seq_group, output)
vllm/entrypoints/llm.py
+29 -24
@@ -381,19 +381,20 @@ class LLM:
381 considered legacy and may be deprecated in the future. You should
382 instead pass them via the ``inputs`` parameter.
383 """
384 - task = self.llm_engine.model_config.task
385 - if task != "generate":
384 + runner_type = self.llm_engine.model_config.runner_type
385 + if runner_type != "generate":
386 messages = [
387 "LLM.generate() is only supported for (conditional) generation "
388 "models (XForCausalLM, XForConditionalGeneration).",
389 ]
390
391 - supported_tasks = self.llm_engine.model_config.supported_tasks
392 - if "generate" in supported_tasks:
391 + supported_runner_types = self.llm_engine.model_config \
392 + .supported_runner_types
393 + if "generate" in supported_runner_types:
394 messages.append(
394 - "Your model supports the 'generate' task, but is "
395 - f"currently initialized for the '{task}' task. Please "
396 - "initialize the model using `--task generate`.")
395 + "Your model supports the 'generate' runner, but is "
396 + f"currently initialized for the '{runner_type}' runner. "
397 + "Please initialize vLLM using `--task generate`.")
398
399 raise ValueError(" ".join(messages))
400
@@ -793,16 +794,18 @@ class LLM:
794 considered legacy and may be deprecated in the future. You should
795 instead pass them via the ``inputs`` parameter.
796 """
796 - task = self.llm_engine.model_config.task
797 - if task != "embedding":
798 - messages = ["LLM.encode() is only supported for embedding models."]
797 + runner_type = self.llm_engine.model_config.runner_type
798 + if runner_type != "pooling":
799 + messages = ["LLM.encode() is only supported for pooling models."]
800
800 - supported_tasks = self.llm_engine.model_config.supported_tasks
801 - if "embedding" in supported_tasks:
801 + supported_runner_types = self.llm_engine.model_config \
802 + .supported_runner_types
803 + if "pooling" in supported_runner_types:
804 messages.append(
803 - "Your model supports the 'embedding' task, but is "
804 - f"currently initialized for the '{task}' task. Please "
805 - "initialize the model using `--task embedding`.")
805 + "Your model supports the 'pooling' runner, but is "
806 + f"currently initialized for the '{runner_type}' runner. "
807 + "Please initialize vLLM using `--task embed`, "
808 + "`--task classify`, `--task score` etc.")
809
810 raise ValueError(" ".join(messages))
811
@@ -864,21 +867,23 @@ class LLM:
867 A list of ``PoolingRequestOutput`` objects containing the
868 generated scores in the same order as the input prompts.
869 """
867 - task = self.llm_engine.model_config.task
868 - if task != "embedding":
869 - messages = ["LLM.score() is only supported for embedding models."]
870 + runner_type = self.llm_engine.model_config.runner_type
871 + if runner_type != "pooling":
872 + messages = ["LLM.score() is only supported for pooling models."]
873
871 - supported_tasks = self.llm_engine.model_config.supported_tasks
872 - if "embedding" in supported_tasks:
874 + supported_runner_types = self.llm_engine.model_config \
875 + .supported_runner_types
876 + if "pooling" in supported_runner_types:
877 messages.append(
874 - "Your model supports the 'embedding' task, but is "
875 - f"currently initialized for the '{task}' task. Please "
876 - "initialize the model using `--task embedding`.")
878 + "Your model supports the 'pooling' runner, but is "
879 + f"currently initialized for the '{runner_type}' runner. "
880 + "Please initialize vLLM using `--task embed`, "
881 + "`--task classify`, `--task score` etc.")
882
883 raise ValueError(" ".join(messages))
884
885 if not self.llm_engine.model_config.is_cross_encoder:
881 - raise ValueError("Your model does not support the cross encoding")
886 + raise ValueError("Your model does not support cross encoding")
887
888 tokenizer = self.llm_engine.get_tokenizer()
889
vllm/entrypoints/openai/api_server.py
+4 -4
@@ -573,7 +573,7 @@ def init_app_state(
573 enable_auto_tools=args.enable_auto_tool_choice,
574 tool_parser=args.tool_call_parser,
575 enable_prompt_tokens_details=args.enable_prompt_tokens_details,
576 - ) if model_config.task == "generate" else None
576 + ) if model_config.runner_type == "generate" else None
577 state.openai_serving_completion = OpenAIServingCompletion(
578 engine_client,
579 model_config,
@@ -582,7 +582,7 @@ def init_app_state(
582 prompt_adapters=args.prompt_adapters,
583 request_logger=request_logger,
584 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
585 - ) if model_config.task == "generate" else None
585 + ) if model_config.runner_type == "generate" else None
586 state.openai_serving_embedding = OpenAIServingEmbedding(
587 engine_client,
588 model_config,
@@ -590,13 +590,13 @@ def init_app_state(
590 request_logger=request_logger,
591 chat_template=resolved_chat_template,
592 chat_template_content_format=args.chat_template_content_format,
593 - ) if model_config.task == "embedding" else None
593 + ) if model_config.runner_type == "pooling" else None
594 state.openai_serving_scores = OpenAIServingScores(
595 engine_client,
596 model_config,
597 base_model_paths,
598 request_logger=request_logger
599 - ) if (model_config.task == "embedding" \
599 + ) if (model_config.runner_type == "pooling" \
600 and model_config.is_cross_encoder) else None
601 state.openai_serving_tokenization = OpenAIServingTokenization(
602 engine_client,
vllm/entrypoints/openai/run_batch.py
+2 -2
@@ -224,7 +224,7 @@ async def main(args):
224 chat_template=None,
225 chat_template_content_format="auto",
226 enable_prompt_tokens_details=args.enable_prompt_tokens_details,
227 - ) if model_config.task == "generate" else None
227 + ) if model_config.runner_type == "generate" else None
228 openai_serving_embedding = OpenAIServingEmbedding(
229 engine,
230 model_config,
@@ -232,7 +232,7 @@ async def main(args):
232 request_logger=request_logger,
233 chat_template=None,
234 chat_template_content_format="auto",
235 - ) if model_config.task == "embedding" else None
235 + ) if model_config.runner_type == "pooling" else None
236
237 tracker = BatchProgressTracker()
238 logger.info("Reading batch from %s...", args.input_file)
vllm/model_executor/model_loader/utils.py
+1 -1
@@ -35,7 +35,7 @@ def get_model_architecture(
35 architectures = ["QuantMixtralForCausalLM"]
36
37 model_cls, arch = ModelRegistry.resolve_model_cls(architectures)
38 - if model_config.task == "embedding":
38 + if model_config.runner_type == "pooling":
39 model_cls = as_embedding_model(model_cls)
40
41 return model_cls, arch
vllm/v1/engine/core.py
+1 -1
@@ -42,7 +42,7 @@ class EngineCore:
42 executor_class: Type[Executor],
43 usage_context: UsageContext,
44 ):
45 - assert vllm_config.model_config.task != "embedding"
45 + assert vllm_config.model_config.runner_type != "pooling"
46
47 logger.info("Initializing an LLM engine (v%s) with config: %s",
48 VLLM_VERSION, vllm_config)
vllm/worker/cpu_worker.py
+1 -1
@@ -163,7 +163,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
163 not in ["medusa", "mlp_speculator", "eagle"]) \
164 else {"return_hidden_states": True}
165 ModelRunnerClass: Type[CPUModelRunnerBase] = CPUModelRunner
166 - if self.model_config.task == "embedding":
166 + if self.model_config.runner_type == "pooling":
167 ModelRunnerClass = CPUPoolingModelRunner
168 elif self.model_config.is_encoder_decoder:
169 ModelRunnerClass = CPUEncoderDecoderModelRunner
vllm/worker/worker.py
+1 -1
@@ -75,7 +75,7 @@ class Worker(LocalOrDistributedWorkerBase):
75 else {"return_hidden_states": True}
76
77 ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
78 - if model_config.task == "embedding":
78 + if model_config.runner_type == "pooling":
79 ModelRunnerClass = PoolingModelRunner
80 elif self.model_config.is_encoder_decoder:
81 ModelRunnerClass = EncoderDecoderModelRunner