A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Frontend] Separate pooling APIs in offline inference (#11129)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 13, 2024 at 18:40 UTC
eeec9e339005d887e0064f7b3e7771295ecd68e7
21 files changed
+659
-294
.buildkite/test-pipeline.yaml
+5
-2
@@ -181,14 +181,14 @@ steps:
181
commands:
182
- VLLM_USE_V1=1 pytest -v -s v1
183
184
-- label: Examples Test # 15min
184
+- label: Examples Test # 25min
185
working_dir: "/vllm-workspace/examples"
186
#mirror_hardwares: [amd]
187
source_file_dependencies:
188
- vllm/entrypoints
189
- examples/
190
commands:
191
- - pip install awscli tensorizer # for llava example and tensorizer test
191
+ - pip install tensorizer # for tensorizer test
192
- python3 offline_inference.py
193
- python3 cpu_offload.py
194
- python3 offline_inference_chat.py
@@ -198,6 +198,9 @@ steps:
198
- python3 offline_inference_vision_language_multi_image.py
199
- python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
200
- python3 offline_inference_encoder_decoder.py
201
+ - python3 offline_inference_classification.py
202
+ - python3 offline_inference_embedding.py
203
+ - python3 offline_inference_scoring.py
204
- python3 offline_profile.py --model facebook/opt-125m
205
206
- label: Prefix Caching Test # 9min
docs/source/models/pooling_models.rst
+45
-8
@@ -6,7 +6,7 @@ Pooling Models
6
vLLM also supports pooling models, including embedding, reranking and reward models.
7
8
In vLLM, pooling models implement the :class:`~vllm.model_executor.models.VllmModelForPooling` interface.
9
-These models use a :class:`~vllm.model_executor.layers.Pooler` to aggregate the final hidden states of the input
9
+These models use a :class:`~vllm.model_executor.layers.Pooler` to extract the final hidden states of the input
10
before returning them.
11
12
.. note::
@@ -45,20 +45,48 @@ which takes priority over both the model's and Sentence Transformers's defaults.
45
^^^^^^^^^^^^^^
46
47
The :class:`~vllm.LLM.encode` method is available to all pooling models in vLLM.
48
-It returns the aggregated hidden states directly.
48
+It returns the extracted hidden states directly, which is useful for reward models.
49
+
50
+.. code-block:: python
51
+
52
+ llm = LLM(model="Qwen/Qwen2.5-Math-RM-72B", task="reward")
53
+ output, = llm.encode("Hello, my name is")
54
+
55
+ data = output.outputs.data
56
+ print(f"Prompt: {prompt!r} | Data: {data!r}")
57
+
58
+``LLM.embed``
59
+^^^^^^^^^^^^^
60
+
61
+The :class:`~vllm.LLM.embed` method outputs an embedding vector for each prompt.
62
+It is primarily designed for embedding models.
63
64
.. code-block:: python
65
66
llm = LLM(model="intfloat/e5-mistral-7b-instruct", task="embed")
53
- outputs = llm.encode("Hello, my name is")
67
+ output, = llm.embed("Hello, my name is")
68
55
- outputs = model.encode(prompts)
56
- for output in outputs:
57
- embeddings = output.outputs.embedding
58
- print(f"Prompt: {prompt!r}, Embeddings (size={len(embeddings)}: {embeddings!r}")
69
+ embeds = output.outputs.embedding
70
+ print(f"Embeddings: {embeds!r} (size={len(embeds)})")
71
72
A code example can be found in `examples/offline_inference_embedding.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_embedding.py>`_.
73
74
+``LLM.classify``
75
+^^^^^^^^^^^^^^^^
76
+
77
+The :class:`~vllm.LLM.classify` method outputs a probability vector for each prompt.
78
+It is primarily designed for classification models.
79
+
80
+.. code-block:: python
81
+
82
+ llm = LLM(model="jason9693/Qwen2.5-1.5B-apeach", task="classify")
83
+ output, = llm.classify("Hello, my name is")
84
+
85
+ probs = output.outputs.probs
86
+ print(f"Class Probabilities: {probs!r} (size={len(probs)})")
87
+
88
+A code example can be found in `examples/offline_inference_classification.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_classification.py>`_.
89
+
90
``LLM.score``
91
^^^^^^^^^^^^^
92
@@ -71,7 +99,16 @@ These types of models serve as rerankers between candidate query-document pairs
99
vLLM can only perform the model inference component (e.g. embedding, reranking) of RAG.
100
To handle RAG at a higher level, you should use integration frameworks such as `LangChain <https://github.com/langchain-ai/langchain>`_.
101
74
-You can use `these tests <https://github.com/vllm-project/vllm/blob/main/tests/models/embedding/language/test_scoring.py>`_ as reference.
102
+.. code-block:: python
103
+
104
+ llm = LLM(model="BAAI/bge-reranker-v2-m3", task="score")
105
+ output, = llm.score("What is the capital of France?",
106
+ "The capital of Brazil is Brasilia.")
107
+
108
+ score = output.outputs.score
109
+ print(f"Score: {score}")
110
+
111
+A code example can be found in `examples/offline_inference_scoring.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_scoring.py>`_.
112
113
Online Inference
114
----------------
examples/offline_inference_classification.py
new
+28
@@ -0,0 +1,28 @@
1
+from vllm import LLM
2
+
3
+# Sample prompts.
4
+prompts = [
5
+ "Hello, my name is",
6
+ "The president of the United States is",
7
+ "The capital of France is",
8
+ "The future of AI is",
9
+]
10
+
11
+# Create an LLM.
12
+# You should pass task="classify" for classification models
13
+model = LLM(
14
+ model="jason9693/Qwen2.5-1.5B-apeach",
15
+ task="classify",
16
+ enforce_eager=True,
17
+)
18
+
19
+# Generate logits. The output is a list of ClassificationRequestOutputs.
20
+outputs = model.classify(prompts)
21
+
22
+# Print the outputs.
23
+for prompt, output in zip(prompts, outputs):
24
+ probs = output.outputs.probs
25
+ probs_trimmed = ((str(probs[:16])[:-1] +
26
+ ", ...]") if len(probs) > 16 else probs)
27
+ print(f"Prompt: {prompt!r} | "
28
+ f"Class Probabilities: {probs_trimmed} (size={len(probs)})")
examples/offline_inference_embedding.py
+11
-5
@@ -9,14 +9,20 @@ prompts = [
9
]
10
11
# Create an LLM.
12
+# You should pass task="embed" for embedding models
13
model = LLM(
14
model="intfloat/e5-mistral-7b-instruct",
14
- task="embed", # You should pass task="embed" for embedding models
15
+ task="embed",
16
enforce_eager=True,
17
)
18
18
-# Generate embedding. The output is a list of PoolingRequestOutputs.
19
-outputs = model.encode(prompts)
19
+# Generate embedding. The output is a list of EmbeddingRequestOutputs.
20
+outputs = model.embed(prompts)
21
+
22
# Print the outputs.
21
-for output in outputs:
22
- print(output.outputs.embedding) # list of 4096 floats
23
+for prompt, output in zip(prompts, outputs):
24
+ embeds = output.outputs.embedding
25
+ embeds_trimmed = ((str(embeds[:16])[:-1] +
26
+ ", ...]") if len(embeds) > 16 else embeds)
27
+ print(f"Prompt: {prompt!r} | "
28
+ f"Embeddings: {embeds_trimmed} (size={len(embeds)})")
examples/offline_inference_scoring.py
new
+23
@@ -0,0 +1,23 @@
1
+from vllm import LLM
2
+
3
+# Sample prompts.
4
+text_1 = "What is the capital of France?"
5
+texts_2 = [
6
+ "The capital of Brazil is Brasilia.", "The capital of France is Paris."
7
+]
8
+
9
+# Create an LLM.
10
+# You should pass task="score" for cross-encoder models
11
+model = LLM(
12
+ model="BAAI/bge-reranker-v2-m3",
13
+ task="score",
14
+ enforce_eager=True,
15
+)
16
+
17
+# Generate scores. The output is a list of ScoringRequestOutputs.
18
+outputs = model.score(text_1, texts_2)
19
+
20
+# Print the outputs.
21
+for text_2, output in zip(texts_2, outputs):
22
+ score = output.outputs.score
23
+ print(f"Pair: {[text_1, text_2]!r} | Score: {score}")
examples/offline_inference_vision_language_embedding.py
+1
-1
@@ -133,7 +133,7 @@ def run_encode(model: str, modality: QueryModality):
133
if req_data.image is not None:
134
mm_data["image"] = req_data.image
135
136
- outputs = req_data.llm.encode({
136
+ outputs = req_data.llm.embed({
137
"prompt": req_data.prompt,
138
"multi_modal_data": mm_data,
139
})
tests/conftest.py
+7
-11
@@ -719,14 +719,6 @@ class VllmRunner:
719
720
return inputs
721
722
- def classify(self, prompts: List[str]) -> List[str]:
723
- req_outputs = self.model.encode(prompts)
724
- outputs = []
725
- for req_output in req_outputs:
726
- embedding = req_output.outputs.embedding
727
- outputs.append(embedding)
728
- return outputs
729
-
722
def generate(
723
self,
724
prompts: List[str],
@@ -897,6 +889,10 @@ class VllmRunner:
889
returned_outputs.append((token_ids, texts))
890
return returned_outputs
891
892
+ def classify(self, prompts: List[str]) -> List[List[float]]:
893
+ req_outputs = self.model.classify(prompts)
894
+ return [req_output.outputs.probs for req_output in req_outputs]
895
+
896
def encode(
897
self,
898
prompts: List[str],
@@ -909,16 +905,16 @@ class VllmRunner:
905
videos=videos,
906
audios=audios)
907
912
- req_outputs = self.model.encode(inputs)
908
+ req_outputs = self.model.embed(inputs)
909
return [req_output.outputs.embedding for req_output in req_outputs]
910
911
def score(
912
self,
913
text_1: Union[str, List[str]],
914
text_2: Union[str, List[str]],
919
- ) -> List[List[float]]:
915
+ ) -> List[float]:
916
req_outputs = self.model.score(text_1, text_2)
921
- return [req_output.outputs.embedding for req_output in req_outputs]
917
+ return [req_output.outputs.score for req_output in req_outputs]
918
919
def __enter__(self):
920
return self
tests/entrypoints/openai/test_score.py
+5
-5
@@ -39,8 +39,8 @@ async def test_text_1_str_text_2_list(server: RemoteOpenAIServer,
39
assert score.id is not None
40
assert score.data is not None
41
assert len(score.data) == 2
42
- assert score.data[0].score[0] <= 0.01
43
- assert score.data[1].score[0] >= 0.9
42
+ assert score.data[0].score <= 0.01
43
+ assert score.data[1].score >= 0.9
44
45
46
@pytest.mark.asyncio
@@ -67,8 +67,8 @@ async def test_text_1_list_text_2_list(server: RemoteOpenAIServer,
67
assert score.id is not None
68
assert score.data is not None
69
assert len(score.data) == 2
70
- assert score.data[0].score[0] <= 0.01
71
- assert score.data[1].score[0] >= 0.9
70
+ assert score.data[0].score <= 0.01
71
+ assert score.data[1].score >= 0.9
72
73
74
@pytest.mark.asyncio
@@ -90,4 +90,4 @@ async def test_text_1_str_text_2_str(server: RemoteOpenAIServer,
90
assert score.id is not None
91
assert score.data is not None
92
assert len(score.data) == 1
93
- assert score.data[0].score[0] >= 0.9
93
+ assert score.data[0].score >= 0.9
tests/models/embedding/language/test_scoring.py
+5
-5
@@ -42,7 +42,7 @@ def test_llm_1_to_1(vllm_runner, hf_runner, model_name, dtype: str):
42
assert len(vllm_outputs) == 1
43
assert len(hf_outputs) == 1
44
45
- assert math.isclose(hf_outputs[0], vllm_outputs[0][0], rel_tol=0.01)
45
+ assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
46
47
48
@pytest.mark.parametrize("dtype", ["half"])
@@ -63,8 +63,8 @@ def test_llm_1_to_N(vllm_runner, hf_runner, model_name, dtype: str):
63
assert len(vllm_outputs) == 2
64
assert len(hf_outputs) == 2
65
66
- assert math.isclose(hf_outputs[0], vllm_outputs[0][0], rel_tol=0.01)
67
- assert math.isclose(hf_outputs[1], vllm_outputs[1][0], rel_tol=0.01)
66
+ assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
67
+ assert math.isclose(hf_outputs[1], vllm_outputs[1], rel_tol=0.01)
68
69
70
@pytest.mark.parametrize("dtype", ["half"])
@@ -85,5 +85,5 @@ def test_llm_N_to_N(vllm_runner, hf_runner, model_name, dtype: str):
85
assert len(vllm_outputs) == 2
86
assert len(hf_outputs) == 2
87
88
- assert math.isclose(hf_outputs[0], vllm_outputs[0][0], rel_tol=0.01)
89
- assert math.isclose(hf_outputs[1], vllm_outputs[1][0], rel_tol=0.01)
88
+ assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
89
+ assert math.isclose(hf_outputs[1], vllm_outputs[1], rel_tol=0.01)
tests/models/test_oot_registration.py
+2
-3
@@ -2,7 +2,7 @@ import os
2
3
import pytest
4
5
-from vllm import LLM, PoolingParams, SamplingParams
5
+from vllm import LLM, SamplingParams
6
from vllm.assets.image import ImageAsset
7
8
from ..utils import fork_new_process_for_each_test
@@ -36,9 +36,8 @@ def test_oot_registration_text_generation(dummy_opt_path):
36
def test_oot_registration_embedding(dummy_gemma2_embedding_path):
37
os.environ["VLLM_PLUGINS"] = "register_dummy_model"
38
prompts = ["Hello, my name is", "The text does not matter"]
39
- sampling_params = PoolingParams()
39
llm = LLM(model=dummy_gemma2_embedding_path, load_format="dummy")
41
- outputs = llm.encode(prompts, sampling_params)
40
+ outputs = llm.embed(prompts)
41
42
for output in outputs:
43
assert all(v == 0 for v in output.outputs.embedding)
vllm/__init__.py
+11
-25
@@ -7,8 +7,11 @@ from vllm.entrypoints.llm import LLM
7
from vllm.executor.ray_utils import initialize_ray_cluster
8
from vllm.inputs import PromptType, TextPrompt, TokensPrompt
9
from vllm.model_executor.models import ModelRegistry
10
-from vllm.outputs import (CompletionOutput, PoolingOutput,
11
- PoolingRequestOutput, RequestOutput)
10
+from vllm.outputs import (ClassificationOutput, ClassificationRequestOutput,
11
+ CompletionOutput, EmbeddingOutput,
12
+ EmbeddingRequestOutput, PoolingOutput,
13
+ PoolingRequestOutput, RequestOutput, ScoringOutput,
14
+ ScoringRequestOutput)
15
from vllm.pooling_params import PoolingParams
16
from vllm.sampling_params import SamplingParams
17
@@ -27,6 +30,12 @@ __all__ = [
30
"CompletionOutput",
31
"PoolingOutput",
32
"PoolingRequestOutput",
33
+ "EmbeddingOutput",
34
+ "EmbeddingRequestOutput",
35
+ "ClassificationOutput",
36
+ "ClassificationRequestOutput",
37
+ "ScoringOutput",
38
+ "ScoringRequestOutput",
39
"LLMEngine",
40
"EngineArgs",
41
"AsyncLLMEngine",
@@ -34,26 +43,3 @@ __all__ = [
43
"initialize_ray_cluster",
44
"PoolingParams",
45
]
37
-
38
-
39
-def __getattr__(name: str):
40
- import warnings
41
-
42
- if name == "EmbeddingOutput":
43
- msg = ("EmbeddingOutput has been renamed to PoolingOutput. "
44
- "The original name will be removed in an upcoming version.")
45
-
46
- warnings.warn(DeprecationWarning(msg), stacklevel=2)
47
-
48
- return PoolingOutput
49
-
50
- if name == "EmbeddingRequestOutput":
51
- msg = ("EmbeddingRequestOutput has been renamed to "
52
- "PoolingRequestOutput. "
53
- "The original name will be removed in an upcoming version.")
54
-
55
- warnings.warn(DeprecationWarning(msg), stacklevel=2)
56
-
57
- return PoolingRequestOutput
58
-
59
- raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/engine/llm_engine.py
+8
-9
@@ -46,11 +46,10 @@ from vllm.outputs import (PoolingRequestOutput, RequestOutput,
46
from vllm.pooling_params import PoolingParams
47
from vllm.prompt_adapter.request import PromptAdapterRequest
48
from vllm.sampling_params import RequestOutputKind, SamplingParams
49
-from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
50
- ParallelSampleSequenceGroup, Sequence,
51
- SequenceGroup, SequenceGroupBase,
52
- SequenceGroupMetadata, SequenceGroupOutput,
53
- SequenceStatus)
49
+from vllm.sequence import (ExecuteModelRequest, ParallelSampleSequenceGroup,
50
+ PoolingSequenceGroupOutput, Sequence, SequenceGroup,
51
+ SequenceGroupBase, SequenceGroupMetadata,
52
+ SequenceGroupOutput, SequenceStatus)
53
from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
54
init_tracer)
55
from vllm.transformers_utils.config import try_get_generation_config
@@ -966,9 +965,9 @@ class LLMEngine:
965
@staticmethod
966
def _process_sequence_group_outputs(
967
seq_group: SequenceGroup,
969
- outputs: List[EmbeddingSequenceGroupOutput],
968
+ outputs: List[PoolingSequenceGroupOutput],
969
) -> None:
971
- seq_group.embeddings = outputs[0].embeddings
970
+ seq_group.pooled_data = outputs[0].data
971
972
for seq in seq_group.get_seqs():
973
seq.status = SequenceStatus.FINISHED_STOPPED
@@ -1784,8 +1783,8 @@ class LLMEngine:
1783
num_prompt_tokens_iter)
1784
# Spec decode, if enabled, emits specialized metrics from the worker in
1785
# sampler output.
1787
- if model_output and (model_output[0].spec_decode_worker_metrics
1788
- is not None):
1786
+ if model_output and isinstance(model_output[0], SamplerOutput) and (
1787
+ model_output[0].spec_decode_worker_metrics is not None):
1788
spec_decode_metrics = model_output[0].spec_decode_worker_metrics
1789
else:
1790
spec_decode_metrics = None
vllm/entrypoints/llm.py
+128
-15
@@ -26,7 +26,9 @@ from vllm.logger import init_logger
26
from vllm.lora.request import LoRARequest
27
from vllm.model_executor.guided_decoding.guided_fields import (
28
GuidedDecodingRequest, LLMGuidedOptions)
29
-from vllm.outputs import PoolingRequestOutput, RequestOutput
29
+from vllm.outputs import (ClassificationRequestOutput, EmbeddingRequestOutput,
30
+ PoolingRequestOutput, RequestOutput,
31
+ ScoringRequestOutput)
32
from vllm.pooling_params import PoolingParams
33
from vllm.prompt_adapter.request import PromptAdapterRequest
34
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
@@ -120,7 +122,7 @@ class LLM:
122
serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
123
"""
124
123
- DEPRECATE_LEGACY: ClassVar[bool] = False
125
+ DEPRECATE_LEGACY: ClassVar[bool] = True
126
"""A flag to toggle whether to deprecate the legacy generate/encode API."""
127
128
DEPRECATE_INIT_POSARGS: ClassVar[bool] = True
@@ -257,11 +259,14 @@ class LLM:
259
self,
260
prompts: Union[PromptType, Sequence[PromptType]],
261
/,
260
- *,
262
sampling_params: Optional[Union[SamplingParams,
263
Sequence[SamplingParams]]] = None,
264
+ *,
265
use_tqdm: bool = True,
266
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
267
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
268
+ guided_options_request: Optional[Union[LLMGuidedOptions,
269
+ GuidedDecodingRequest]] = None,
270
) -> List[RequestOutput]:
271
...
272
@@ -275,6 +280,9 @@ class LLM:
280
prompt_token_ids: Optional[List[int]] = None,
281
use_tqdm: bool = True,
282
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
283
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
284
+ guided_options_request: Optional[Union[LLMGuidedOptions,
285
+ GuidedDecodingRequest]] = None,
286
) -> List[RequestOutput]:
287
...
288
@@ -288,6 +296,9 @@ class LLM:
296
prompt_token_ids: Optional[List[List[int]]] = None,
297
use_tqdm: bool = True,
298
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
299
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
300
+ guided_options_request: Optional[Union[LLMGuidedOptions,
301
+ GuidedDecodingRequest]] = None,
302
) -> List[RequestOutput]:
303
...
304
@@ -302,6 +313,9 @@ class LLM:
313
prompt_token_ids: List[int],
314
use_tqdm: bool = True,
315
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
316
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
317
+ guided_options_request: Optional[Union[LLMGuidedOptions,
318
+ GuidedDecodingRequest]] = None,
319
) -> List[RequestOutput]:
320
...
321
@@ -316,6 +330,9 @@ class LLM:
330
prompt_token_ids: List[List[int]],
331
use_tqdm: bool = True,
332
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
333
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
334
+ guided_options_request: Optional[Union[LLMGuidedOptions,
335
+ GuidedDecodingRequest]] = None,
336
) -> List[RequestOutput]:
337
...
338
@@ -328,6 +345,9 @@ class LLM:
345
prompt_token_ids: Union[List[int], List[List[int]]],
346
use_tqdm: bool = True,
347
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
348
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
349
+ guided_options_request: Optional[Union[LLMGuidedOptions,
350
+ GuidedDecodingRequest]] = None,
351
) -> List[RequestOutput]:
352
...
353
@@ -678,11 +698,12 @@ class LLM:
698
self,
699
prompts: Union[PromptType, Sequence[PromptType]],
700
/,
681
- *,
701
pooling_params: Optional[Union[PoolingParams,
702
Sequence[PoolingParams]]] = None,
703
+ *,
704
use_tqdm: bool = True,
705
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
706
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
707
) -> List[PoolingRequestOutput]:
708
...
709
@@ -696,6 +717,7 @@ class LLM:
717
prompt_token_ids: Optional[List[int]] = None,
718
use_tqdm: bool = True,
719
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
720
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
721
) -> List[PoolingRequestOutput]:
722
...
723
@@ -709,6 +731,7 @@ class LLM:
731
prompt_token_ids: Optional[List[List[int]]] = None,
732
use_tqdm: bool = True,
733
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
734
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
735
) -> List[PoolingRequestOutput]:
736
...
737
@@ -723,6 +746,7 @@ class LLM:
746
prompt_token_ids: List[int],
747
use_tqdm: bool = True,
748
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
749
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
750
) -> List[PoolingRequestOutput]:
751
...
752
@@ -737,6 +761,7 @@ class LLM:
761
prompt_token_ids: List[List[int]],
762
use_tqdm: bool = True,
763
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
764
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
765
) -> List[PoolingRequestOutput]:
766
...
767
@@ -749,6 +774,7 @@ class LLM:
774
prompt_token_ids: Union[List[int], List[List[int]]],
775
use_tqdm: bool = True,
776
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
777
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
778
) -> List[PoolingRequestOutput]:
779
...
780
@@ -768,7 +794,8 @@ class LLM:
794
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
795
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
796
) -> List[PoolingRequestOutput]:
771
- """Generates the completions for the input prompts.
797
+ """Apply pooling to the hidden states corresponding to the input
798
+ prompts.
799
800
This class automatically batches the given prompts, considering
801
the memory constraint. For the best performance, put all of your prompts
@@ -787,7 +814,7 @@ class LLM:
814
815
Returns:
816
A list of ``PoolingRequestOutput`` objects containing the
790
- generated embeddings in the same order as the input prompts.
817
+ pooled hidden states in the same order as the input prompts.
818
819
Note:
820
Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
@@ -833,28 +860,110 @@ class LLM:
860
return self.engine_class.validate_outputs(outputs,
861
PoolingRequestOutput)
862
863
+ def embed(
864
+ self,
865
+ prompts: Union[PromptType, Sequence[PromptType]],
866
+ /,
867
+ *,
868
+ use_tqdm: bool = True,
869
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
870
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
871
+ ) -> List[EmbeddingRequestOutput]:
872
+ """
873
+ Generate an embedding vector for each prompt.
874
+
875
+ This class automatically batches the given prompts, considering
876
+ the memory constraint. For the best performance, put all of your prompts
877
+ into a single list and pass it to this method.
878
+
879
+ Args:
880
+ prompts: The prompts to the LLM. You may pass a sequence of prompts
881
+ for batch inference. See :class:`~vllm.inputs.PromptType`
882
+ for more details about the format of each prompts.
883
+ use_tqdm: Whether to use tqdm to display the progress bar.
884
+ lora_request: LoRA request to use for generation, if any.
885
+ prompt_adapter_request: Prompt Adapter request to use for
886
+ generation, if any.
887
+
888
+ Returns:
889
+ A list of ``EmbeddingRequestOutput`` objects containing the
890
+ embedding vectors in the same order as the input prompts.
891
+ """
892
+ if self.llm_engine.model_config.task != "embed":
893
+ raise ValueError(
894
+ "Embedding API is only enabled for `--task embed`")
895
+
896
+ items = self.encode(prompts,
897
+ use_tqdm=use_tqdm,
898
+ lora_request=lora_request,
899
+ prompt_adapter_request=prompt_adapter_request)
900
+
901
+ return [EmbeddingRequestOutput.from_base(item) for item in items]
902
+
903
+ def classify(
904
+ self,
905
+ prompts: Union[PromptType, Sequence[PromptType]],
906
+ /,
907
+ *,
908
+ use_tqdm: bool = True,
909
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
910
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
911
+ ) -> List[ClassificationRequestOutput]:
912
+ """
913
+ Generate class logits for each prompt.
914
+
915
+ This class automatically batches the given prompts, considering
916
+ the memory constraint. For the best performance, put all of your prompts
917
+ into a single list and pass it to this method.
918
+
919
+ Args:
920
+ prompts: The prompts to the LLM. You may pass a sequence of prompts
921
+ for batch inference. See :class:`~vllm.inputs.PromptType`
922
+ for more details about the format of each prompts.
923
+ use_tqdm: Whether to use tqdm to display the progress bar.
924
+ lora_request: LoRA request to use for generation, if any.
925
+ prompt_adapter_request: Prompt Adapter request to use for
926
+ generation, if any.
927
+
928
+ Returns:
929
+ A list of ``ClassificationRequestOutput`` objects containing the
930
+ embedding vectors in the same order as the input prompts.
931
+ """
932
+ if self.llm_engine.model_config.task != "classify":
933
+ raise ValueError(
934
+ "Classification API is only enabled for `--task classify`")
935
+
936
+ items = self.encode(prompts,
937
+ use_tqdm=use_tqdm,
938
+ lora_request=lora_request,
939
+ prompt_adapter_request=prompt_adapter_request)
940
+
941
+ return [ClassificationRequestOutput.from_base(item) for item in items]
942
+
943
def score(
944
self,
945
text_1: Union[SingletonPrompt, Sequence[SingletonPrompt]],
946
text_2: Union[SingletonPrompt, Sequence[SingletonPrompt]],
947
/,
948
+ *,
949
truncate_prompt_tokens: Optional[int] = None,
950
use_tqdm: bool = True,
951
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
952
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
845
- ) -> List[PoolingRequestOutput]:
846
- """Generates similarity scores for all pairs <text,text_pair>.
953
+ ) -> List[ScoringRequestOutput]:
954
+ """Generate similarity scores for all pairs ``<text,text_pair>``.
955
848
- The inputs can be 1 -> 1, 1 -> N or N -> N. In the 1 - N case
849
- the text_1 sentence will be replicated N times to pair with the text_2
850
- sentences. The input pairs are used to build a list of prompts for the
956
+ The inputs can be ``1 -> 1``, ``1 -> N`` or ``N -> N``.
957
+ In the ``1 - N`` case the ``text_1`` sentence will be replicated ``N``
958
+ times to pair with the ``text_2`` sentences.
959
+ The input pairs are used to build a list of prompts for the
960
cross encoder model. This class automatically batches the prompts,
961
considering the memory constraint. For the best performance, put all
962
of your texts into a single list and pass it to this method.
963
964
Args:
965
text_1: can be a single prompt or a list of prompts, in which
857
- case it has to have the same length as the text_2 list
966
+ case it has to have the same length as the ``text_2`` list
967
text_2: The texts to pair with the query to form the input
968
to the LLM. See :class:`~vllm.inputs.PromptType` for
969
more details about the format of each prompts.
@@ -864,7 +973,7 @@ class LLM:
973
generation, if any.
974
975
Returns:
867
- A list of ``PoolingRequestOutput`` objects containing the
976
+ A list of ``ScoringRequestOutput`` objects containing the
977
generated scores in the same order as the input prompts.
978
"""
979
runner_type = self.llm_engine.model_config.runner_type
@@ -884,6 +993,8 @@ class LLM:
993
994
if not self.llm_engine.model_config.is_cross_encoder:
995
raise ValueError("Your model does not support cross encoding")
996
+ if self.llm_engine.model_config.task != "score":
997
+ raise ValueError("Score API is only enabled for `--task score`")
998
999
tokenizer = self.llm_engine.get_tokenizer()
1000
@@ -954,8 +1065,10 @@ class LLM:
1065
)
1066
1067
outputs = self._run_engine(use_tqdm=use_tqdm)
957
- return self.engine_class.validate_outputs(outputs,
958
- PoolingRequestOutput)
1068
+ items = self.engine_class.validate_outputs(outputs,
1069
+ PoolingRequestOutput)
1070
+
1071
+ return [ScoringRequestOutput.from_base(item) for item in items]
1072
1073
def start_profile(self) -> None:
1074
self.llm_engine.start_profile()
vllm/entrypoints/openai/protocol.py
+1
-1
@@ -900,7 +900,7 @@ class EmbeddingResponse(OpenAIBaseModel):
900
class ScoreResponseData(OpenAIBaseModel):
901
index: int
902
object: str = "score"
903
- score: Union[List[float], str]
903
+ score: float
904
905
906
class ScoreResponse(OpenAIBaseModel):
vllm/entrypoints/openai/serving_embedding.py
+6
-3
@@ -18,14 +18,15 @@ from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
18
ErrorResponse, UsageInfo)
19
from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
20
from vllm.logger import init_logger
21
-from vllm.outputs import PoolingOutput, PoolingRequestOutput
21
+from vllm.outputs import (EmbeddingOutput, EmbeddingRequestOutput,
22
+ PoolingRequestOutput)
23
from vllm.utils import merge_async_iterators
24
25
logger = init_logger(__name__)
26
27
28
def _get_embedding(
28
- output: PoolingOutput,
29
+ output: EmbeddingOutput,
30
encoding_format: Literal["float", "base64"],
31
) -> Union[List[float], str]:
32
if encoding_format == "float":
@@ -46,8 +47,10 @@ def request_output_to_embedding_response(
47
data: List[EmbeddingResponseData] = []
48
num_prompt_tokens = 0
49
for idx, final_res in enumerate(final_res_batch):
50
+ embedding_res = EmbeddingRequestOutput.from_base(final_res)
51
prompt_token_ids = final_res.prompt_token_ids
50
- embedding = _get_embedding(final_res.outputs, encoding_format)
52
+
53
+ embedding = _get_embedding(embedding_res.outputs, encoding_format)
54
embedding_data = EmbeddingResponseData(index=idx, embedding=embedding)
55
data.append(embedding_data)
56
vllm/entrypoints/openai/serving_engine.py
+7
-5
@@ -31,7 +31,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
31
ErrorResponse,
32
LoadLoraAdapterRequest,
33
ModelCard, ModelList,
34
- ModelPermission,
34
+ ModelPermission, ScoreRequest,
35
TokenizeChatRequest,
36
TokenizeCompletionRequest,
37
UnloadLoraAdapterRequest)
@@ -73,7 +73,7 @@ class LoRAModulePath:
73
74
75
CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
76
- EmbeddingCompletionRequest,
76
+ EmbeddingCompletionRequest, ScoreRequest,
77
TokenizeCompletionRequest]
78
79
ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,
@@ -567,12 +567,14 @@ class OpenAIServing:
567
return None
568
569
@staticmethod
570
- def _base_request_id(raw_request: Request,
570
+ def _base_request_id(raw_request: Optional[Request],
571
default: Optional[str] = None) -> Optional[str]:
572
"""Pulls the request id to use from a header, if provided"""
573
default = default or random_uuid()
574
- return raw_request.headers.get(
575
- "X-Request-Id", default) if raw_request is not None else default
574
+ if raw_request is None:
575
+ return default
576
+
577
+ return raw_request.headers.get("X-Request-Id", default)
578
579
@staticmethod
580
def _get_decoded_token(logprob: Logprob,
vllm/entrypoints/openai/serving_score.py
+6
-6
@@ -13,7 +13,7 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse, ScoreRequest,
13
from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
14
from vllm.inputs.data import TokensPrompt
15
from vllm.logger import init_logger
16
-from vllm.outputs import PoolingRequestOutput
16
+from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
17
from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
18
from vllm.utils import make_async, merge_async_iterators
19
@@ -24,13 +24,13 @@ def request_output_to_score_response(
24
final_res_batch: List[PoolingRequestOutput], request_id: str,
25
created_time: int, model_name: str) -> ScoreResponse:
26
data: List[ScoreResponseData] = []
27
- score = None
27
num_prompt_tokens = 0
28
for idx, final_res in enumerate(final_res_batch):
30
- if final_res is not None:
31
- score = final_res.outputs.embedding
32
- score_data = ScoreResponseData(index=idx, score=score)
33
- data.append(score_data)
29
+ classify_res = ScoringRequestOutput.from_base(final_res)
30
+
31
+ score_data = ScoreResponseData(index=idx,
32
+ score=classify_res.outputs.score)
33
+ data.append(score_data)
34
35
usage = UsageInfo(
36
prompt_tokens=num_prompt_tokens,
vllm/model_executor/layers/pooler.py
+196
-92
@@ -1,14 +1,16 @@
1
from enum import IntEnum
2
-from typing import List, Optional
2
+from typing import List, Optional, Union
3
4
import torch
5
import torch.nn as nn
6
+import torch.nn.functional as F
7
from transformers import PretrainedConfig
8
+from typing_extensions import assert_never
9
10
from vllm.config import PoolerConfig
11
from vllm.model_executor.pooling_metadata import (PoolingMetadata,
12
PoolingTensors)
11
-from vllm.sequence import EmbeddingSequenceGroupOutput, PoolerOutput
13
+from vllm.sequence import PoolerOutput, PoolingSequenceGroupOutput
14
from vllm.transformers_utils.config import (
15
get_cross_encoder_activation_function)
16
@@ -22,7 +24,7 @@ class PoolingType(IntEnum):
24
MEAN = 4
25
26
25
-class Pooler(nn.Module):
27
+class SimplePooler(nn.Module):
28
"""A layer that pools specific information from hidden states.
29
30
This layer does the following:
@@ -35,21 +37,203 @@ class Pooler(nn.Module):
37
normalize: Whether to normalize the pooled data.
38
"""
39
40
+ @staticmethod
41
+ def from_pooling_type(
42
+ pooling_type: PoolingType,
43
+ *,
44
+ normalize: bool,
45
+ softmax: bool,
46
+ step_tag_id: Optional[int] = None,
47
+ returned_token_ids: Optional[List[int]] = None,
48
+ ) -> "SimplePooler":
49
+ if pooling_type == PoolingType.LAST:
50
+ assert step_tag_id is None and returned_token_ids is None
51
+ return LastPool(normalize=normalize, softmax=softmax)
52
+ if pooling_type == PoolingType.ALL:
53
+ assert step_tag_id is None and returned_token_ids is None
54
+ return AllPool(normalize=normalize, softmax=softmax)
55
+ if pooling_type == PoolingType.CLS:
56
+ assert step_tag_id is None and returned_token_ids is None
57
+ return CLSPool(normalize=normalize, softmax=softmax)
58
+ if pooling_type == PoolingType.MEAN:
59
+ assert step_tag_id is None and returned_token_ids is None
60
+ return MeanPool(normalize=normalize, softmax=softmax)
61
+ if pooling_type == PoolingType.STEP:
62
+ return StepPool(normalize=normalize,
63
+ softmax=softmax,
64
+ step_tag_id=step_tag_id,
65
+ returned_token_ids=returned_token_ids)
66
+
67
+ assert_never(pooling_type)
68
+
69
+ def __init__(self, *, normalize: bool, softmax: bool) -> None:
70
+ super().__init__()
71
+
72
+ self.head = PoolerHead(normalize=normalize, softmax=softmax)
73
+
74
+ def get_prompt_lens(
75
+ self,
76
+ hidden_states: torch.Tensor,
77
+ pooling_metadata: PoolingMetadata,
78
+ ) -> torch.Tensor:
79
+ return PoolingTensors.from_pooling_metadata(
80
+ pooling_metadata, hidden_states.device).prompt_lens
81
+
82
+ def extract_states(
83
+ self,
84
+ hidden_states: torch.Tensor,
85
+ pooling_metadata: PoolingMetadata,
86
+ ) -> Union[list[torch.Tensor], torch.Tensor]:
87
+ raise NotImplementedError
88
+
89
+ def build_output(self, data: torch.Tensor) -> PoolingSequenceGroupOutput:
90
+ return PoolingSequenceGroupOutput(data)
91
+
92
+ def forward(
93
+ self,
94
+ hidden_states: torch.Tensor,
95
+ pooling_metadata: PoolingMetadata,
96
+ ) -> PoolerOutput:
97
+ pooled_data = self.extract_states(hidden_states, pooling_metadata)
98
+ pooled_data = self.head(pooled_data)
99
+ pooled_outputs = [self.build_output(data) for data in pooled_data]
100
+ return PoolerOutput(outputs=pooled_outputs)
101
+
102
+
103
+class CLSPool(SimplePooler):
104
+
105
+ def extract_states(
106
+ self,
107
+ hidden_states: torch.Tensor,
108
+ pooling_metadata: PoolingMetadata,
109
+ ) -> Union[list[torch.Tensor], torch.Tensor]:
110
+ prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
111
+
112
+ first_token_flat_indices = torch.zeros_like(prompt_lens)
113
+ first_token_flat_indices[1:] += torch.cumsum(prompt_lens, dim=0)[:-1]
114
+ return hidden_states[first_token_flat_indices]
115
+
116
+
117
+class LastPool(SimplePooler):
118
+
119
+ def extract_states(
120
+ self,
121
+ hidden_states: torch.Tensor,
122
+ pooling_metadata: PoolingMetadata,
123
+ ) -> Union[list[torch.Tensor], torch.Tensor]:
124
+ prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
125
+
126
+ last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1
127
+ return hidden_states[last_token_flat_indices]
128
+
129
+
130
+class AllPool(SimplePooler):
131
+
132
+ def extract_states(
133
+ self,
134
+ hidden_states: torch.Tensor,
135
+ pooling_metadata: PoolingMetadata,
136
+ ) -> Union[list[torch.Tensor], torch.Tensor]:
137
+ prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
138
+
139
+ offset = 0
140
+ pooled_data = list[torch.Tensor]()
141
+ for prompt_len in prompt_lens:
142
+ pooled_data.append(hidden_states[offset:offset + prompt_len])
143
+ offset += prompt_len
144
+
145
+ return pooled_data
146
+
147
+
148
+class MeanPool(SimplePooler):
149
+
150
+ def extract_states(
151
+ self,
152
+ hidden_states: torch.Tensor,
153
+ pooling_metadata: PoolingMetadata,
154
+ ) -> Union[list[torch.Tensor], torch.Tensor]:
155
+ prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
156
+
157
+ cumsum = torch.cumsum(hidden_states, dim=0)
158
+ start_indices = torch.cat([
159
+ torch.tensor([0], device=hidden_states.device),
160
+ torch.cumsum(prompt_lens[:-1], dim=0)
161
+ ])
162
+ end_indices = torch.cumsum(prompt_lens, dim=0)
163
+ return (cumsum[end_indices - 1] - cumsum[start_indices] +
164
+ hidden_states[start_indices]) / prompt_lens.unsqueeze(1)
165
+
166
+
167
+class StepPool(SimplePooler):
168
+
169
def __init__(
170
self,
40
- pooling_type: PoolingType,
171
+ *,
172
normalize: bool,
173
softmax: bool,
174
step_tag_id: Optional[int] = None,
175
returned_token_ids: Optional[List[int]] = None,
176
):
177
+ super().__init__(normalize=normalize, softmax=softmax)
178
+
179
+ self.step_tag_id = step_tag_id
180
+ self.returned_token_ids = returned_token_ids
181
+
182
+ def extract_states(
183
+ self,
184
+ hidden_states: torch.Tensor,
185
+ pooling_metadata: PoolingMetadata,
186
+ ) -> Union[list[torch.Tensor], torch.Tensor]:
187
+ prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
188
+
189
+ returned_token_ids = self.returned_token_ids
190
+ if returned_token_ids is not None and len(returned_token_ids) > 0:
191
+ hidden_states = hidden_states[:, returned_token_ids]
192
+
193
+ step_tag_id = self.step_tag_id
194
+
195
+ offset = 0
196
+ pooled_data = list[torch.Tensor]()
197
+ for prompt_len, seq_data_i in zip(prompt_lens,
198
+ pooling_metadata.seq_data.values()):
199
+ pooled_data_i = hidden_states[offset:offset + prompt_len]
200
+ if step_tag_id is not None:
201
+ token_ids = torch.tensor(seq_data_i.prompt_token_ids)
202
+ pooled_data_i = pooled_data_i[token_ids == step_tag_id]
203
+
204
+ offset += prompt_len
205
+ pooled_data.append(pooled_data_i)
206
+
207
+ return pooled_data
208
+
209
+
210
+class PoolerHead(nn.Module):
211
+
212
+ def __init__(self, *, normalize: bool, softmax: bool) -> None:
213
super().__init__()
214
48
- self.pooling_type = pooling_type
215
self.normalize = normalize
216
self.softmax = softmax
51
- self.step_tag_id = step_tag_id
52
- self.returned_token_ids = returned_token_ids
217
+
218
+ def forward(self, pooled_data: Union[list[torch.Tensor], torch.Tensor]):
219
+ if self.normalize:
220
+ if isinstance(pooled_data, list):
221
+ pooled_data = [
222
+ F.normalize(data, p=2, dim=1) for data in pooled_data
223
+ ]
224
+ else:
225
+ pooled_data = F.normalize(pooled_data, p=2, dim=1)
226
+
227
+ if self.softmax:
228
+ if isinstance(pooled_data, list):
229
+ pooled_data = [F.softmax(data, dim=-1) for data in pooled_data]
230
+ else:
231
+ pooled_data = F.softmax(pooled_data, dim=-1)
232
+
233
+ return pooled_data
234
+
235
+
236
+class Pooler(nn.Module):
237
238
@classmethod
239
def from_config_with_defaults(
@@ -60,8 +244,8 @@ class Pooler(nn.Module):
244
softmax: bool,
245
step_tag_id: Optional[int] = None,
246
returned_token_ids: Optional[List[int]] = None,
63
- ) -> "Pooler":
64
- return cls(
247
+ ) -> SimplePooler:
248
+ return SimplePooler.from_pooling_type(
249
pooling_type=PoolingType[pooler_config.pooling_type]
250
if pooler_config.pooling_type is not None else pooling_type,
251
normalize=pooler_config.normalize
@@ -75,85 +259,6 @@ class Pooler(nn.Module):
259
returned_token_ids,
260
)
261
78
- def forward(
79
- self,
80
- hidden_states: torch.Tensor,
81
- pooling_metadata: PoolingMetadata,
82
- ) -> PoolerOutput:
83
- """Pools specific information from hidden states based on metadata."""
84
-
85
- prompt_lens = PoolingTensors.from_pooling_metadata(
86
- pooling_metadata, hidden_states.device).prompt_lens
87
-
88
- if self.pooling_type is PoolingType.CLS:
89
- first_token_flat_indices = torch.zeros_like(prompt_lens)
90
- first_token_flat_indices[1:] += torch.cumsum(prompt_lens,
91
- dim=0)[:-1]
92
- pooled_data = hidden_states[first_token_flat_indices]
93
- elif self.pooling_type == PoolingType.LAST:
94
- last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1
95
- pooled_data = hidden_states[last_token_flat_indices]
96
- elif self.pooling_type == PoolingType.ALL:
97
- offset = 0
98
- pooled_data = []
99
- for prompt_len in prompt_lens:
100
- pooled_data.append(hidden_states[offset:offset + prompt_len])
101
- offset += prompt_len
102
- elif self.pooling_type == PoolingType.MEAN:
103
- # Calculate mean pooling
104
- cumsum = torch.cumsum(hidden_states, dim=0)
105
- start_indices = torch.cat([
106
- torch.tensor([0], device=hidden_states.device),
107
- torch.cumsum(prompt_lens[:-1], dim=0)
108
- ])
109
- end_indices = torch.cumsum(prompt_lens, dim=0)
110
- pooled_data = (
111
- cumsum[end_indices - 1] - cumsum[start_indices] +
112
- hidden_states[start_indices]) / prompt_lens.unsqueeze(1)
113
- elif self.pooling_type == PoolingType.STEP:
114
- returned_token_ids = self.returned_token_ids
115
- if returned_token_ids is not None and len(returned_token_ids) > 0:
116
- hidden_states = hidden_states[:, returned_token_ids]
117
-
118
- step_tag_id = self.step_tag_id
119
-
120
- offset = 0
121
- pooled_data = []
122
- for prompt_len, seq_data_i in zip(
123
- prompt_lens, pooling_metadata.seq_data.values()):
124
- pooled_data_i = hidden_states[offset:offset + prompt_len]
125
- if step_tag_id is not None:
126
- token_ids = torch.tensor(seq_data_i.prompt_token_ids)
127
- pooled_data_i = pooled_data_i[token_ids == step_tag_id]
128
-
129
- offset += prompt_len
130
- pooled_data.append(pooled_data_i)
131
- else:
132
- raise ValueError(f"Invalid pooling type: {self.pooling_type}")
133
-
134
- if self.normalize:
135
- if isinstance(pooled_data, list):
136
- pooled_data = [
137
- nn.functional.normalize(data, p=2, dim=1)
138
- for data in pooled_data
139
- ]
140
- else:
141
- pooled_data = nn.functional.normalize(pooled_data, p=2, dim=1)
142
-
143
- if self.softmax:
144
- if isinstance(pooled_data, list):
145
- pooled_data = [
146
- nn.functional.softmax(data, dim=-1) for data in pooled_data
147
- ]
148
- else:
149
- pooled_data = nn.functional.softmax(pooled_data, dim=-1)
150
-
151
- pooled_outputs = [
152
- EmbeddingSequenceGroupOutput(data.tolist()) for data in pooled_data
153
- ]
154
-
155
- return PoolerOutput(outputs=pooled_outputs)
156
-
262
263
class CrossEncodingPooler(nn.Module):
264
"""A layer that pools specific information from hidden states.
@@ -208,9 +313,8 @@ class CrossEncodingPooler(nn.Module):
313
if self.pooler is not None:
314
# apply classifier once on the full batch if possible
315
pooled_output = self.classifier(pooled_output)
211
- logits = self.default_activation_function(pooled_output)
316
213
- pooled_outputs = [
214
- EmbeddingSequenceGroupOutput(data.tolist()) for data in logits
215
- ]
317
+ scores = self.default_activation_function(pooled_output).squeeze(-1)
318
+
319
+ pooled_outputs = [PoolingSequenceGroupOutput(data) for data in scores]
320
return PoolerOutput(outputs=pooled_outputs)
vllm/model_executor/models/gritlm.py
+9
-6
@@ -2,19 +2,20 @@ from array import array
2
from typing import List, Optional, Union
3
4
import torch
5
-from torch import nn
5
+import torch.nn as nn
6
from xformers.ops.fmha.attn_bias import BlockDiagonalMask
7
8
from vllm.attention import AttentionMetadata
9
from vllm.attention.backends.xformers import XFormersImpl
10
from vllm.config import ModelConfig, VllmConfig
11
from vllm.logger import init_logger
12
+from vllm.model_executor.layers.pooler import PoolerHead
13
from vllm.model_executor.models.llama import LlamaForCausalLM
14
from vllm.model_executor.pooling_metadata import (PoolingMetadata,
15
PoolingTensors)
16
from vllm.multimodal.utils import cached_get_tokenizer
16
-from vllm.sequence import (EmbeddingSequenceGroupOutput, IntermediateTensors,
17
- PoolerOutput)
17
+from vllm.sequence import (IntermediateTensors, PoolerOutput,
18
+ PoolingSequenceGroupOutput)
19
20
logger = init_logger(__name__)
21
@@ -52,6 +53,8 @@ class GritLMPooler(nn.Module):
53
self.embed_pattern_ids = tokens_to_ids(
54
["▁<", "|", "embed", "|", ">", "<0x0A>"])
55
56
+ self.head = PoolerHead(normalize=True, softmax=False)
57
+
58
def _find_array(self, arr: array, target: array, start_idx: int) -> int:
59
"""
60
Find the first occurrence of target in arr starting from start_idx.
@@ -75,7 +78,7 @@ class GritLMPooler(nn.Module):
78
return i
79
return -1
80
78
- def _get_instruction_len(self, prompt_token_ids: array) -> bool:
81
+ def _get_instruction_len(self, prompt_token_ids: array) -> int:
82
"""
83
Get the length of the instruction in the prompt.
84
@@ -168,10 +171,10 @@ class GritLMPooler(nn.Module):
171
mean_embeddings = sum_embeddings / num_non_instruction_tokens.unsqueeze(
172
1)
173
171
- pooled_data = nn.functional.normalize(mean_embeddings, p=2, dim=1)
174
+ pooled_data = self.head(mean_embeddings)
175
176
pooled_outputs = [
174
- EmbeddingSequenceGroupOutput(data.tolist()) for data in pooled_data
177
+ PoolingSequenceGroupOutput(data) for data in pooled_data
178
]
179
180
return PoolerOutput(outputs=pooled_outputs)
vllm/outputs.py
+137
-70
@@ -1,9 +1,13 @@
1
import time
2
+import warnings
3
from dataclasses import dataclass
3
-from typing import Dict, List, Optional
4
+from typing import Dict, Generic, List, Optional
5
from typing import Sequence as GenericSequence
6
from typing import Union
7
8
+import torch
9
+from typing_extensions import TypeVar
10
+
11
from vllm.lora.request import LoRARequest
12
from vllm.multimodal.inputs import MultiModalPlaceholderDict
13
from vllm.sampling_params import RequestOutputKind
@@ -57,14 +61,26 @@ class PoolingOutput:
61
"""The output data of one pooling output of a request.
62
63
Args:
60
- embedding: The embedding vector, which is a list of floats. The
61
- length of vector depends on the model as listed in the embedding guide.
64
+ data: The extracted hidden states.
65
"""
63
- embedding: List[float]
66
+ data: torch.Tensor
67
68
def __repr__(self) -> str:
66
- return (f"PoolingOutput("
67
- f"embedding={len(self.embedding)})")
69
+ return (f"PoolingOutput(data={self.data})")
70
+
71
+ def __eq__(self, other: object) -> bool:
72
+ return (isinstance(other, self.__class__) and bool(
73
+ (self.data == other.data).all()))
74
+
75
+ @property
76
+ def embedding(self) -> list[float]:
77
+ msg = ("`LLM.encode()` now returns raw outputs. "
78
+ "To return embeddings, use `LLM.embed()`. "
79
+ "To return class probabilities, use `LLM.classify()` "
80
+ "and access the `probs` attribute. ")
81
+ warnings.warn(msg, DeprecationWarning, stacklevel=2)
82
+
83
+ return self.data.tolist()
84
85
86
class RequestOutput:
@@ -316,7 +332,10 @@ class RequestOutput:
332
f"multi_modal_placeholders={self.multi_modal_placeholders})")
333
334
319
-class PoolingRequestOutput:
335
+_O = TypeVar("_O", default=PoolingOutput)
336
+
337
+
338
+class PoolingRequestOutput(Generic[_O]):
339
"""
340
The output data of a pooling request to the LLM.
341
@@ -327,24 +346,24 @@ class PoolingRequestOutput:
346
finished (bool): A flag indicating whether the pooling is completed.
347
"""
348
330
- def __init__(self, request_id: str, outputs: "PoolingOutput",
349
+ def __init__(self, request_id: str, outputs: _O,
350
prompt_token_ids: List[int], finished: bool):
351
self.request_id = request_id
352
self.prompt_token_ids = prompt_token_ids
353
self.finished = finished
354
self.outputs = outputs
355
337
- @classmethod
338
- def from_seq_group(cls,
339
- seq_group: 'SequenceGroup') -> "PoolingRequestOutput":
340
- if seq_group.embeddings is None:
341
- raise ValueError(
342
- "Embeddings are missing in seq_group for EmbeddingRequest.")
343
- output = PoolingOutput(seq_group.embeddings)
356
+ @staticmethod
357
+ def from_seq_group(seq_group: SequenceGroup) -> "PoolingRequestOutput":
358
+ pooled_data = seq_group.pooled_data
359
+ assert pooled_data is not None
360
+
361
+ output = PoolingOutput(pooled_data)
362
prompt_token_ids = seq_group.prompt_token_ids
363
finished = seq_group.is_finished()
364
347
- return cls(seq_group.request_id, output, prompt_token_ids, finished)
365
+ return PoolingRequestOutput(seq_group.request_id, output,
366
+ prompt_token_ids, finished)
367
368
def __repr__(self):
369
"""
@@ -356,89 +375,137 @@ class PoolingRequestOutput:
375
Returns:
376
str: A string representation of the PoolingRequestOutput instance.
377
"""
359
- return (f"PoolingRequestOutput(request_id='{self.request_id}', "
360
- f"outputs={repr(self.outputs)}, "
378
+ return (f"{type(self).__name__}(request_id={self.request_id!r}, "
379
+ f"outputs={self.outputs!r}, "
380
f"prompt_token_ids={self.prompt_token_ids}, "
381
f"finished={self.finished})")
382
383
384
+class RequestOutputFactory:
385
+
386
+ @staticmethod
387
+ def create(seq_group: SequenceGroup,
388
+ seq_id_to_seq_group: Dict[str, SequenceGroupBase],
389
+ use_cache: bool = False):
390
+ if seq_group.pooled_data is not None:
391
+ return PoolingRequestOutput.from_seq_group(seq_group)
392
+ else:
393
+ return RequestOutput.from_seq_group(seq_group, use_cache,
394
+ seq_id_to_seq_group)
395
+
396
+
397
@dataclass
366
-class ScoreOutput:
367
- """The output data of one completion output of a request.
398
+class EmbeddingOutput:
399
+ """The output data of one embedding output of a request.
400
401
Args:
370
- score: The score, which is a list of floats.
371
- index: The correspondent text index of the score.
402
+ embedding: The embedding vector, which is a list of floats.
403
+ Its length depends on the hidden dimension of the model.
404
"""
373
- index: int
374
- score: List[float]
405
+ embedding: list[float]
406
+
407
+ @staticmethod
408
+ def from_base(pooling_output: PoolingOutput):
409
+ pooled_data = pooling_output.data
410
+ if pooled_data.ndim != 1:
411
+ raise ValueError("pooled_data should be a 1-D embedding vector")
412
+
413
+ return EmbeddingOutput(pooled_data.tolist())
414
+
415
+ @property
416
+ def hidden_size(self) -> int:
417
+ return len(self.embedding)
418
419
def __repr__(self) -> str:
377
- return (f"ScoreOutput("
378
- f"score={self.score}), "
379
- f"index={self.index})")
420
+ return f"EmbeddingOutput(hidden_size={self.hidden_size})"
421
422
382
-class ScoreRequestOutput:
383
- """
384
- The output data of an score request to the LLM.
423
+class EmbeddingRequestOutput(PoolingRequestOutput[EmbeddingOutput]):
424
+
425
+ @staticmethod
426
+ def from_base(request_output: PoolingRequestOutput):
427
+ return EmbeddingRequestOutput(
428
+ request_id=request_output.request_id,
429
+ outputs=EmbeddingOutput.from_base(request_output.outputs),
430
+ prompt_token_ids=request_output.prompt_token_ids,
431
+ finished=request_output.finished,
432
+ )
433
+
434
+
435
+@dataclass
436
+class ClassificationOutput:
437
+ """The output data of one classification output of a request.
438
439
Args:
387
- request_id (str): A unique identifier for the score request.
388
- outputs (score): The embedding results for the given input.
440
+ probs: The probability vector, which is a list of floats.
441
+ Its length depends on the number of classes.
442
"""
443
+ probs: list[float]
444
391
- def __init__(self, request_id: str, outputs: "ScoreOutput"):
392
- self.request_id = request_id
393
- self.outputs = outputs
445
+ @staticmethod
446
+ def from_base(pooling_output: PoolingOutput):
447
+ pooled_data = pooling_output.data
448
+ if pooled_data.ndim != 1:
449
+ raise ValueError("pooled_data should be a 1-D probability vector")
450
395
- def __repr__(self):
396
- """
397
- Returns a string representation of an ScoreRequestOutput instance.
451
+ return ClassificationOutput(pooled_data.tolist())
452
399
- The representation includes the request_id and the number of outputs,
400
- providing a quick overview of the embedding request's results.
453
+ @property
454
+ def num_classes(self) -> int:
455
+ return len(self.probs)
456
402
- Returns:
403
- str: A string representation of the ScoreRequestOutput instance.
404
- """
405
- return (f"ScoreRequestOutput(request_id='{self.request_id}', "
406
- f"outputs={repr(self.outputs)}")
457
+ def __repr__(self) -> str:
458
+ return f"ClassificationOutput(num_classes={self.num_classes})"
459
460
409
-class RequestOutputFactory:
461
+class ClassificationRequestOutput(PoolingRequestOutput[ClassificationOutput]):
462
463
@staticmethod
412
- def create(seq_group: SequenceGroup,
413
- seq_id_to_seq_group: Dict[str, SequenceGroupBase],
414
- use_cache: bool = False):
415
- # Determine the type based on a condition, for example:
416
- if hasattr(seq_group,
417
- 'embeddings') and seq_group.embeddings is not None:
418
- return PoolingRequestOutput.from_seq_group(seq_group)
419
- else:
420
- return RequestOutput.from_seq_group(seq_group, use_cache,
421
- seq_id_to_seq_group)
464
+ def from_base(request_output: PoolingRequestOutput):
465
+ return ClassificationRequestOutput(
466
+ request_id=request_output.request_id,
467
+ outputs=ClassificationOutput.from_base(request_output.outputs),
468
+ prompt_token_ids=request_output.prompt_token_ids,
469
+ finished=request_output.finished,
470
+ )
471
472
424
-def __getattr__(name: str):
425
- import warnings
473
+@dataclass
474
+class ScoringOutput:
475
+ """The output data of one scoring output of a request.
476
427
- if name == "EmbeddingOutput":
428
- msg = ("EmbeddingOutput has been renamed to PoolingOutput. "
429
- "The original name will be removed in an upcoming version.")
477
+ Args:
478
+ score: The similarity score, which is a scalar value.
479
+ """
480
+ score: float
481
+
482
+ @staticmethod
483
+ def from_base(pooling_output: PoolingOutput):
484
+ pooled_data = pooling_output.data
485
+ if pooled_data.ndim != 0:
486
+ raise ValueError("pooled_data should be a scalar score")
487
431
- warnings.warn(DeprecationWarning(msg), stacklevel=2)
488
+ return ScoringOutput(pooled_data.item())
489
433
- return PoolingOutput
490
+ def __repr__(self) -> str:
491
+ return f"ScoringOutput(score={self.score})"
492
435
- if name == "EmbeddingRequestOutput":
436
- msg = ("EmbeddingRequestOutput has been renamed to "
437
- "PoolingRequestOutput. "
438
- "The original name will be removed in an upcoming version.")
493
+ @property
494
+ def embedding(self) -> list[float]:
495
+ msg = ("`LLM.score()` now returns scalar scores. "
496
+ "Please access it via the `score` attribute. ")
497
+ warnings.warn(msg, DeprecationWarning, stacklevel=2)
498
440
- warnings.warn(DeprecationWarning(msg), stacklevel=2)
499
+ return [self.score]
500
442
- return PoolingRequestOutput
501
444
- raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
502
+class ScoringRequestOutput(PoolingRequestOutput[ScoringOutput]):
503
+
504
+ @staticmethod
505
+ def from_base(request_output: PoolingRequestOutput):
506
+ return ScoringRequestOutput(
507
+ request_id=request_output.request_id,
508
+ outputs=ScoringOutput.from_base(request_output.outputs),
509
+ prompt_token_ids=request_output.prompt_token_ids,
510
+ finished=request_output.finished,
511
+ )
vllm/sequence.py
+18
-22
@@ -617,10 +617,9 @@ class SequenceGroup:
617
sampling_params: The sampling parameters used to generate the outputs.
618
arrival_time: The arrival time of the request.
619
lora_request: LoRA request.
620
- embeddings: The embeddings vectors of the prompt of the sequence group
621
- for a pooling model.
622
- pooling_params: The pooling parameters used to generate the pooling
620
+ pooling_params: The parameters used to generate the pooler
621
for a pooling model.
622
+ pooled_data: The extracted hidden states from a pooling model.
623
encoder_seq: Optional, the single encoder sequence. Should be None
624
unless you are working with an encoder/decoder model.
625
trace_headers: OpenTelemetry trace headers.
@@ -635,8 +634,8 @@ class SequenceGroup:
634
arrival_time: float,
635
sampling_params: Optional[SamplingParams] = None,
636
lora_request: Optional[LoRARequest] = None,
638
- embeddings: Optional[List[float]] = None,
637
pooling_params: Optional[PoolingParams] = None,
638
+ pooled_data: Optional[torch.Tensor] = None,
639
encoder_seq: Optional[Sequence] = None,
640
trace_headers: Optional[Mapping[str, str]] = None,
641
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -658,8 +657,8 @@ class SequenceGroup:
657
self.lora_request = lora_request
658
self.prompt_logprobs: Optional[PromptLogprobs] = None
659
self.state = SequenceGroupState()
661
- self.embeddings = embeddings
660
self.pooling_params = pooling_params
661
+ self.pooled_data = pooled_data
662
self.prompt_adapter_request = prompt_adapter_request
663
self.encoder_seq = encoder_seq
664
self.trace_headers = trace_headers
@@ -1033,8 +1032,8 @@ class CompletionSequenceGroupOutput(
1032
msgspec.Struct,
1033
omit_defaults=True, # type: ignore[call-arg]
1034
array_like=True): # type: ignore[call-arg]
1036
- __metaclass__ = SequenceGroupOutput
1035
"""The model output associated with a completion sequence group."""
1036
+ __metaclass__ = SequenceGroupOutput
1037
samples: List[SequenceOutput]
1038
# Prompt logprob for each prompt query token.
1039
prompt_logprobs: Optional[PromptLogprobs]
@@ -1050,23 +1049,24 @@ class CompletionSequenceGroupOutput(
1049
and self.prompt_logprobs == other.prompt_logprobs)
1050
1051
1053
-class EmbeddingSequenceGroupOutput(
1052
+class PoolingSequenceGroupOutput(
1053
msgspec.Struct,
1054
omit_defaults=True, # type: ignore[call-arg]
1055
array_like=True, # type: ignore[call-arg]
1056
):
1058
- """The model output associated with an embedding sequence group."""
1057
+ """The model output associated with a pooling sequence group."""
1058
__metaclass__ = SequenceGroupOutput
1060
- embeddings: List[int]
1059
+ # Annotated as Any to be compatible with msgspec
1060
+ # The actual type is in SequenceGroup.pooled_data
1061
+ data: Any
1062
1063
def __repr__(self) -> str:
1063
- return (f"EmbeddingSequenceGroupOutput("
1064
- f"embeddings_shape={len(self.embeddings)})")
1064
+ return f"PoolingSequenceGroupOutput(data={self.data}"
1065
1066
def __eq__(self, other: object) -> bool:
1067
- if not isinstance(other, EmbeddingSequenceGroupOutput):
1067
+ if not isinstance(other, PoolingSequenceGroupOutput):
1068
raise NotImplementedError()
1069
- return self.embeddings == other.embeddings
1069
+ return self.data == other.data
1070
1071
1072
# cannot use msgspec.Struct here because Dynamo does not support it
@@ -1085,7 +1085,7 @@ class IntermediateTensors:
1085
elif isinstance(key, slice):
1086
return self.__class__({k: v[key] for k, v in self.tensors.items()})
1087
1088
- def __setitem__(self, key: str, value):
1088
+ def __setitem__(self, key: str, value: torch.Tensor):
1089
self.tensors[key] = value
1090
1091
def __len__(self):
@@ -1103,16 +1103,12 @@ class PoolerOutput(
1103
omit_defaults=True, # type: ignore[call-arg]
1104
array_like=True): # type: ignore[call-arg]
1105
"""The output from a pooling operation in the pooling model."""
1106
- outputs: List[EmbeddingSequenceGroupOutput]
1107
-
1108
- # lazy import to avoid circular import
1109
- from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
1110
- spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1106
+ outputs: List[PoolingSequenceGroupOutput]
1107
1112
- def __getitem__(self, idx: int) -> EmbeddingSequenceGroupOutput:
1108
+ def __getitem__(self, idx: int) -> PoolingSequenceGroupOutput:
1109
return self.outputs[idx]
1110
1115
- def __setitem__(self, idx: int, value):
1111
+ def __setitem__(self, idx: int, value: PoolingSequenceGroupOutput):
1112
self.outputs[idx] = value
1113
1114
def __len__(self):
@@ -1385,8 +1381,8 @@ class ParallelSampleSequenceGroup(SequenceGroupBase):
1381
arrival_time=seq_group.arrival_time,
1382
sampling_params=original_params,
1383
lora_request=seq_group.lora_request,
1388
- embeddings=seq_group.embeddings,
1384
pooling_params=seq_group.pooling_params,
1385
+ pooled_data=seq_group.pooled_data,
1386
encoder_seq=seq_group.encoder_seq,
1387
trace_headers=seq_group.trace_headers,
1388
prompt_adapter_request=seq_group.prompt_adapter_request,