@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eeec9e3390
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Frontend] Separate pooling APIs in offline inference (#11129)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 13, 2024 at 18:40 UTC eeec9e339005d887e0064f7b3e7771295ecd68e7
21 files changed +659 -294
.buildkite/test-pipeline.yaml
+5 -2
@@ -181,14 +181,14 @@ steps:
181 commands:
182 - VLLM_USE_V1=1 pytest -v -s v1
183
184 -- label: Examples Test # 15min
184 +- label: Examples Test # 25min
185 working_dir: "/vllm-workspace/examples"
186 #mirror_hardwares: [amd]
187 source_file_dependencies:
188 - vllm/entrypoints
189 - examples/
190 commands:
191 - - pip install awscli tensorizer # for llava example and tensorizer test
191 + - pip install tensorizer # for tensorizer test
192 - python3 offline_inference.py
193 - python3 cpu_offload.py
194 - python3 offline_inference_chat.py
@@ -198,6 +198,9 @@ steps:
198 - python3 offline_inference_vision_language_multi_image.py
199 - python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
200 - python3 offline_inference_encoder_decoder.py
201 + - python3 offline_inference_classification.py
202 + - python3 offline_inference_embedding.py
203 + - python3 offline_inference_scoring.py
204 - python3 offline_profile.py --model facebook/opt-125m
205
206 - label: Prefix Caching Test # 9min
docs/source/models/pooling_models.rst
+45 -8
@@ -6,7 +6,7 @@ Pooling Models
6 vLLM also supports pooling models, including embedding, reranking and reward models.
7
8 In vLLM, pooling models implement the :class:`~vllm.model_executor.models.VllmModelForPooling` interface.
9 -These models use a :class:`~vllm.model_executor.layers.Pooler` to aggregate the final hidden states of the input
9 +These models use a :class:`~vllm.model_executor.layers.Pooler` to extract the final hidden states of the input
10 before returning them.
11
12 .. note::
@@ -45,20 +45,48 @@ which takes priority over both the model's and Sentence Transformers's defaults.
45 ^^^^^^^^^^^^^^
46
47 The :class:`~vllm.LLM.encode` method is available to all pooling models in vLLM.
48 -It returns the aggregated hidden states directly.
48 +It returns the extracted hidden states directly, which is useful for reward models.
49 +
50 +.. code-block:: python
51 +
52 + llm = LLM(model="Qwen/Qwen2.5-Math-RM-72B", task="reward")
53 + output, = llm.encode("Hello, my name is")
54 +
55 + data = output.outputs.data
56 + print(f"Prompt: {prompt!r} | Data: {data!r}")
57 +
58 +``LLM.embed``
59 +^^^^^^^^^^^^^
60 +
61 +The :class:`~vllm.LLM.embed` method outputs an embedding vector for each prompt.
62 +It is primarily designed for embedding models.
63
64 .. code-block:: python
65
66 llm = LLM(model="intfloat/e5-mistral-7b-instruct", task="embed")
53 - outputs = llm.encode("Hello, my name is")
67 + output, = llm.embed("Hello, my name is")
68
55 - outputs = model.encode(prompts)
56 - for output in outputs:
57 - embeddings = output.outputs.embedding
58 - print(f"Prompt: {prompt!r}, Embeddings (size={len(embeddings)}: {embeddings!r}")
69 + embeds = output.outputs.embedding
70 + print(f"Embeddings: {embeds!r} (size={len(embeds)})")
71
72 A code example can be found in `examples/offline_inference_embedding.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_embedding.py>`_.
73
74 +``LLM.classify``
75 +^^^^^^^^^^^^^^^^
76 +
77 +The :class:`~vllm.LLM.classify` method outputs a probability vector for each prompt.
78 +It is primarily designed for classification models.
79 +
80 +.. code-block:: python
81 +
82 + llm = LLM(model="jason9693/Qwen2.5-1.5B-apeach", task="classify")
83 + output, = llm.classify("Hello, my name is")
84 +
85 + probs = output.outputs.probs
86 + print(f"Class Probabilities: {probs!r} (size={len(probs)})")
87 +
88 +A code example can be found in `examples/offline_inference_classification.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_classification.py>`_.
89 +
90 ``LLM.score``
91 ^^^^^^^^^^^^^
92
@@ -71,7 +99,16 @@ These types of models serve as rerankers between candidate query-document pairs
99 vLLM can only perform the model inference component (e.g. embedding, reranking) of RAG.
100 To handle RAG at a higher level, you should use integration frameworks such as `LangChain <https://github.com/langchain-ai/langchain>`_.
101
74 -You can use `these tests <https://github.com/vllm-project/vllm/blob/main/tests/models/embedding/language/test_scoring.py>`_ as reference.
102 +.. code-block:: python
103 +
104 + llm = LLM(model="BAAI/bge-reranker-v2-m3", task="score")
105 + output, = llm.score("What is the capital of France?",
106 + "The capital of Brazil is Brasilia.")
107 +
108 + score = output.outputs.score
109 + print(f"Score: {score}")
110 +
111 +A code example can be found in `examples/offline_inference_scoring.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_scoring.py>`_.
112
113 Online Inference
114 ----------------
examples/offline_inference_classification.py new
+28
@@ -0,0 +1,28 @@
1 +from vllm import LLM
2 +
3 +# Sample prompts.
4 +prompts = [
5 + "Hello, my name is",
6 + "The president of the United States is",
7 + "The capital of France is",
8 + "The future of AI is",
9 +]
10 +
11 +# Create an LLM.
12 +# You should pass task="classify" for classification models
13 +model = LLM(
14 + model="jason9693/Qwen2.5-1.5B-apeach",
15 + task="classify",
16 + enforce_eager=True,
17 +)
18 +
19 +# Generate logits. The output is a list of ClassificationRequestOutputs.
20 +outputs = model.classify(prompts)
21 +
22 +# Print the outputs.
23 +for prompt, output in zip(prompts, outputs):
24 + probs = output.outputs.probs
25 + probs_trimmed = ((str(probs[:16])[:-1] +
26 + ", ...]") if len(probs) > 16 else probs)
27 + print(f"Prompt: {prompt!r} | "
28 + f"Class Probabilities: {probs_trimmed} (size={len(probs)})")
examples/offline_inference_embedding.py
+11 -5
@@ -9,14 +9,20 @@ prompts = [
9 ]
10
11 # Create an LLM.
12 +# You should pass task="embed" for embedding models
13 model = LLM(
14 model="intfloat/e5-mistral-7b-instruct",
14 - task="embed", # You should pass task="embed" for embedding models
15 + task="embed",
16 enforce_eager=True,
17 )
18
18 -# Generate embedding. The output is a list of PoolingRequestOutputs.
19 -outputs = model.encode(prompts)
19 +# Generate embedding. The output is a list of EmbeddingRequestOutputs.
20 +outputs = model.embed(prompts)
21 +
22 # Print the outputs.
21 -for output in outputs:
22 - print(output.outputs.embedding) # list of 4096 floats
23 +for prompt, output in zip(prompts, outputs):
24 + embeds = output.outputs.embedding
25 + embeds_trimmed = ((str(embeds[:16])[:-1] +
26 + ", ...]") if len(embeds) > 16 else embeds)
27 + print(f"Prompt: {prompt!r} | "
28 + f"Embeddings: {embeds_trimmed} (size={len(embeds)})")
examples/offline_inference_scoring.py new
+23
@@ -0,0 +1,23 @@
1 +from vllm import LLM
2 +
3 +# Sample prompts.
4 +text_1 = "What is the capital of France?"
5 +texts_2 = [
6 + "The capital of Brazil is Brasilia.", "The capital of France is Paris."
7 +]
8 +
9 +# Create an LLM.
10 +# You should pass task="score" for cross-encoder models
11 +model = LLM(
12 + model="BAAI/bge-reranker-v2-m3",
13 + task="score",
14 + enforce_eager=True,
15 +)
16 +
17 +# Generate scores. The output is a list of ScoringRequestOutputs.
18 +outputs = model.score(text_1, texts_2)
19 +
20 +# Print the outputs.
21 +for text_2, output in zip(texts_2, outputs):
22 + score = output.outputs.score
23 + print(f"Pair: {[text_1, text_2]!r} | Score: {score}")
examples/offline_inference_vision_language_embedding.py
+1 -1
@@ -133,7 +133,7 @@ def run_encode(model: str, modality: QueryModality):
133 if req_data.image is not None:
134 mm_data["image"] = req_data.image
135
136 - outputs = req_data.llm.encode({
136 + outputs = req_data.llm.embed({
137 "prompt": req_data.prompt,
138 "multi_modal_data": mm_data,
139 })
tests/conftest.py
+7 -11
@@ -719,14 +719,6 @@ class VllmRunner:
719
720 return inputs
721
722 - def classify(self, prompts: List[str]) -> List[str]:
723 - req_outputs = self.model.encode(prompts)
724 - outputs = []
725 - for req_output in req_outputs:
726 - embedding = req_output.outputs.embedding
727 - outputs.append(embedding)
728 - return outputs
729 -
722 def generate(
723 self,
724 prompts: List[str],
@@ -897,6 +889,10 @@ class VllmRunner:
889 returned_outputs.append((token_ids, texts))
890 return returned_outputs
891
892 + def classify(self, prompts: List[str]) -> List[List[float]]:
893 + req_outputs = self.model.classify(prompts)
894 + return [req_output.outputs.probs for req_output in req_outputs]
895 +
896 def encode(
897 self,
898 prompts: List[str],
@@ -909,16 +905,16 @@ class VllmRunner:
905 videos=videos,
906 audios=audios)
907
912 - req_outputs = self.model.encode(inputs)
908 + req_outputs = self.model.embed(inputs)
909 return [req_output.outputs.embedding for req_output in req_outputs]
910
911 def score(
912 self,
913 text_1: Union[str, List[str]],
914 text_2: Union[str, List[str]],
919 - ) -> List[List[float]]:
915 + ) -> List[float]:
916 req_outputs = self.model.score(text_1, text_2)
921 - return [req_output.outputs.embedding for req_output in req_outputs]
917 + return [req_output.outputs.score for req_output in req_outputs]
918
919 def __enter__(self):
920 return self
tests/entrypoints/openai/test_score.py
+5 -5
@@ -39,8 +39,8 @@ async def test_text_1_str_text_2_list(server: RemoteOpenAIServer,
39 assert score.id is not None
40 assert score.data is not None
41 assert len(score.data) == 2
42 - assert score.data[0].score[0] <= 0.01
43 - assert score.data[1].score[0] >= 0.9
42 + assert score.data[0].score <= 0.01
43 + assert score.data[1].score >= 0.9
44
45
46 @pytest.mark.asyncio
@@ -67,8 +67,8 @@ async def test_text_1_list_text_2_list(server: RemoteOpenAIServer,
67 assert score.id is not None
68 assert score.data is not None
69 assert len(score.data) == 2
70 - assert score.data[0].score[0] <= 0.01
71 - assert score.data[1].score[0] >= 0.9
70 + assert score.data[0].score <= 0.01
71 + assert score.data[1].score >= 0.9
72
73
74 @pytest.mark.asyncio
@@ -90,4 +90,4 @@ async def test_text_1_str_text_2_str(server: RemoteOpenAIServer,
90 assert score.id is not None
91 assert score.data is not None
92 assert len(score.data) == 1
93 - assert score.data[0].score[0] >= 0.9
93 + assert score.data[0].score >= 0.9
tests/models/embedding/language/test_scoring.py
+5 -5
@@ -42,7 +42,7 @@ def test_llm_1_to_1(vllm_runner, hf_runner, model_name, dtype: str):
42 assert len(vllm_outputs) == 1
43 assert len(hf_outputs) == 1
44
45 - assert math.isclose(hf_outputs[0], vllm_outputs[0][0], rel_tol=0.01)
45 + assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
46
47
48 @pytest.mark.parametrize("dtype", ["half"])
@@ -63,8 +63,8 @@ def test_llm_1_to_N(vllm_runner, hf_runner, model_name, dtype: str):
63 assert len(vllm_outputs) == 2
64 assert len(hf_outputs) == 2
65
66 - assert math.isclose(hf_outputs[0], vllm_outputs[0][0], rel_tol=0.01)
67 - assert math.isclose(hf_outputs[1], vllm_outputs[1][0], rel_tol=0.01)
66 + assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
67 + assert math.isclose(hf_outputs[1], vllm_outputs[1], rel_tol=0.01)
68
69
70 @pytest.mark.parametrize("dtype", ["half"])
@@ -85,5 +85,5 @@ def test_llm_N_to_N(vllm_runner, hf_runner, model_name, dtype: str):
85 assert len(vllm_outputs) == 2
86 assert len(hf_outputs) == 2
87
88 - assert math.isclose(hf_outputs[0], vllm_outputs[0][0], rel_tol=0.01)
89 - assert math.isclose(hf_outputs[1], vllm_outputs[1][0], rel_tol=0.01)
88 + assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
89 + assert math.isclose(hf_outputs[1], vllm_outputs[1], rel_tol=0.01)
tests/models/test_oot_registration.py
+2 -3
@@ -2,7 +2,7 @@ import os
2
3 import pytest
4
5 -from vllm import LLM, PoolingParams, SamplingParams
5 +from vllm import LLM, SamplingParams
6 from vllm.assets.image import ImageAsset
7
8 from ..utils import fork_new_process_for_each_test
@@ -36,9 +36,8 @@ def test_oot_registration_text_generation(dummy_opt_path):
36 def test_oot_registration_embedding(dummy_gemma2_embedding_path):
37 os.environ["VLLM_PLUGINS"] = "register_dummy_model"
38 prompts = ["Hello, my name is", "The text does not matter"]
39 - sampling_params = PoolingParams()
39 llm = LLM(model=dummy_gemma2_embedding_path, load_format="dummy")
41 - outputs = llm.encode(prompts, sampling_params)
40 + outputs = llm.embed(prompts)
41
42 for output in outputs:
43 assert all(v == 0 for v in output.outputs.embedding)
vllm/__init__.py
+11 -25
@@ -7,8 +7,11 @@ from vllm.entrypoints.llm import LLM
7 from vllm.executor.ray_utils import initialize_ray_cluster
8 from vllm.inputs import PromptType, TextPrompt, TokensPrompt
9 from vllm.model_executor.models import ModelRegistry
10 -from vllm.outputs import (CompletionOutput, PoolingOutput,
11 - PoolingRequestOutput, RequestOutput)
10 +from vllm.outputs import (ClassificationOutput, ClassificationRequestOutput,
11 + CompletionOutput, EmbeddingOutput,
12 + EmbeddingRequestOutput, PoolingOutput,
13 + PoolingRequestOutput, RequestOutput, ScoringOutput,
14 + ScoringRequestOutput)
15 from vllm.pooling_params import PoolingParams
16 from vllm.sampling_params import SamplingParams
17
@@ -27,6 +30,12 @@ __all__ = [
30 "CompletionOutput",
31 "PoolingOutput",
32 "PoolingRequestOutput",
33 + "EmbeddingOutput",
34 + "EmbeddingRequestOutput",
35 + "ClassificationOutput",
36 + "ClassificationRequestOutput",
37 + "ScoringOutput",
38 + "ScoringRequestOutput",
39 "LLMEngine",
40 "EngineArgs",
41 "AsyncLLMEngine",
@@ -34,26 +43,3 @@ __all__ = [
43 "initialize_ray_cluster",
44 "PoolingParams",
45 ]
37 -
38 -
39 -def __getattr__(name: str):
40 - import warnings
41 -
42 - if name == "EmbeddingOutput":
43 - msg = ("EmbeddingOutput has been renamed to PoolingOutput. "
44 - "The original name will be removed in an upcoming version.")
45 -
46 - warnings.warn(DeprecationWarning(msg), stacklevel=2)
47 -
48 - return PoolingOutput
49 -
50 - if name == "EmbeddingRequestOutput":
51 - msg = ("EmbeddingRequestOutput has been renamed to "
52 - "PoolingRequestOutput. "
53 - "The original name will be removed in an upcoming version.")
54 -
55 - warnings.warn(DeprecationWarning(msg), stacklevel=2)
56 -
57 - return PoolingRequestOutput
58 -
59 - raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/engine/llm_engine.py
+8 -9
@@ -46,11 +46,10 @@ from vllm.outputs import (PoolingRequestOutput, RequestOutput,
46 from vllm.pooling_params import PoolingParams
47 from vllm.prompt_adapter.request import PromptAdapterRequest
48 from vllm.sampling_params import RequestOutputKind, SamplingParams
49 -from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
50 - ParallelSampleSequenceGroup, Sequence,
51 - SequenceGroup, SequenceGroupBase,
52 - SequenceGroupMetadata, SequenceGroupOutput,
53 - SequenceStatus)
49 +from vllm.sequence import (ExecuteModelRequest, ParallelSampleSequenceGroup,
50 + PoolingSequenceGroupOutput, Sequence, SequenceGroup,
51 + SequenceGroupBase, SequenceGroupMetadata,
52 + SequenceGroupOutput, SequenceStatus)
53 from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
54 init_tracer)
55 from vllm.transformers_utils.config import try_get_generation_config
@@ -966,9 +965,9 @@ class LLMEngine:
965 @staticmethod
966 def _process_sequence_group_outputs(
967 seq_group: SequenceGroup,
969 - outputs: List[EmbeddingSequenceGroupOutput],
968 + outputs: List[PoolingSequenceGroupOutput],
969 ) -> None:
971 - seq_group.embeddings = outputs[0].embeddings
970 + seq_group.pooled_data = outputs[0].data
971
972 for seq in seq_group.get_seqs():
973 seq.status = SequenceStatus.FINISHED_STOPPED
@@ -1784,8 +1783,8 @@ class LLMEngine:
1783 num_prompt_tokens_iter)
1784 # Spec decode, if enabled, emits specialized metrics from the worker in
1785 # sampler output.
1787 - if model_output and (model_output[0].spec_decode_worker_metrics
1788 - is not None):
1786 + if model_output and isinstance(model_output[0], SamplerOutput) and (
1787 + model_output[0].spec_decode_worker_metrics is not None):
1788 spec_decode_metrics = model_output[0].spec_decode_worker_metrics
1789 else:
1790 spec_decode_metrics = None
vllm/entrypoints/llm.py
+128 -15
@@ -26,7 +26,9 @@ from vllm.logger import init_logger
26 from vllm.lora.request import LoRARequest
27 from vllm.model_executor.guided_decoding.guided_fields import (
28 GuidedDecodingRequest, LLMGuidedOptions)
29 -from vllm.outputs import PoolingRequestOutput, RequestOutput
29 +from vllm.outputs import (ClassificationRequestOutput, EmbeddingRequestOutput,
30 + PoolingRequestOutput, RequestOutput,
31 + ScoringRequestOutput)
32 from vllm.pooling_params import PoolingParams
33 from vllm.prompt_adapter.request import PromptAdapterRequest
34 from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
@@ -120,7 +122,7 @@ class LLM:
122 serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
123 """
124
123 - DEPRECATE_LEGACY: ClassVar[bool] = False
125 + DEPRECATE_LEGACY: ClassVar[bool] = True
126 """A flag to toggle whether to deprecate the legacy generate/encode API."""
127
128 DEPRECATE_INIT_POSARGS: ClassVar[bool] = True
@@ -257,11 +259,14 @@ class LLM:
259 self,
260 prompts: Union[PromptType, Sequence[PromptType]],
261 /,
260 - *,
262 sampling_params: Optional[Union[SamplingParams,
263 Sequence[SamplingParams]]] = None,
264 + *,
265 use_tqdm: bool = True,
266 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
267 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
268 + guided_options_request: Optional[Union[LLMGuidedOptions,
269 + GuidedDecodingRequest]] = None,
270 ) -> List[RequestOutput]:
271 ...
272
@@ -275,6 +280,9 @@ class LLM:
280 prompt_token_ids: Optional[List[int]] = None,
281 use_tqdm: bool = True,
282 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
283 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
284 + guided_options_request: Optional[Union[LLMGuidedOptions,
285 + GuidedDecodingRequest]] = None,
286 ) -> List[RequestOutput]:
287 ...
288
@@ -288,6 +296,9 @@ class LLM:
296 prompt_token_ids: Optional[List[List[int]]] = None,
297 use_tqdm: bool = True,
298 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
299 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
300 + guided_options_request: Optional[Union[LLMGuidedOptions,
301 + GuidedDecodingRequest]] = None,
302 ) -> List[RequestOutput]:
303 ...
304
@@ -302,6 +313,9 @@ class LLM:
313 prompt_token_ids: List[int],
314 use_tqdm: bool = True,
315 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
316 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
317 + guided_options_request: Optional[Union[LLMGuidedOptions,
318 + GuidedDecodingRequest]] = None,
319 ) -> List[RequestOutput]:
320 ...
321
@@ -316,6 +330,9 @@ class LLM:
330 prompt_token_ids: List[List[int]],
331 use_tqdm: bool = True,
332 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
333 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
334 + guided_options_request: Optional[Union[LLMGuidedOptions,
335 + GuidedDecodingRequest]] = None,
336 ) -> List[RequestOutput]:
337 ...
338
@@ -328,6 +345,9 @@ class LLM:
345 prompt_token_ids: Union[List[int], List[List[int]]],
346 use_tqdm: bool = True,
347 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
348 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
349 + guided_options_request: Optional[Union[LLMGuidedOptions,
350 + GuidedDecodingRequest]] = None,
351 ) -> List[RequestOutput]:
352 ...
353
@@ -678,11 +698,12 @@ class LLM:
698 self,
699 prompts: Union[PromptType, Sequence[PromptType]],
700 /,
681 - *,
701 pooling_params: Optional[Union[PoolingParams,
702 Sequence[PoolingParams]]] = None,
703 + *,
704 use_tqdm: bool = True,
705 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
706 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
707 ) -> List[PoolingRequestOutput]:
708 ...
709
@@ -696,6 +717,7 @@ class LLM:
717 prompt_token_ids: Optional[List[int]] = None,
718 use_tqdm: bool = True,
719 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
720 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
721 ) -> List[PoolingRequestOutput]:
722 ...
723
@@ -709,6 +731,7 @@ class LLM:
731 prompt_token_ids: Optional[List[List[int]]] = None,
732 use_tqdm: bool = True,
733 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
734 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
735 ) -> List[PoolingRequestOutput]:
736 ...
737
@@ -723,6 +746,7 @@ class LLM:
746 prompt_token_ids: List[int],
747 use_tqdm: bool = True,
748 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
749 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
750 ) -> List[PoolingRequestOutput]:
751 ...
752
@@ -737,6 +761,7 @@ class LLM:
761 prompt_token_ids: List[List[int]],
762 use_tqdm: bool = True,
763 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
764 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
765 ) -> List[PoolingRequestOutput]:
766 ...
767
@@ -749,6 +774,7 @@ class LLM:
774 prompt_token_ids: Union[List[int], List[List[int]]],
775 use_tqdm: bool = True,
776 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
777 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
778 ) -> List[PoolingRequestOutput]:
779 ...
780
@@ -768,7 +794,8 @@ class LLM:
794 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
795 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
796 ) -> List[PoolingRequestOutput]:
771 - """Generates the completions for the input prompts.
797 + """Apply pooling to the hidden states corresponding to the input
798 + prompts.
799
800 This class automatically batches the given prompts, considering
801 the memory constraint. For the best performance, put all of your prompts
@@ -787,7 +814,7 @@ class LLM:
814
815 Returns:
816 A list of ``PoolingRequestOutput`` objects containing the
790 - generated embeddings in the same order as the input prompts.
817 + pooled hidden states in the same order as the input prompts.
818
819 Note:
820 Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
@@ -833,28 +860,110 @@ class LLM:
860 return self.engine_class.validate_outputs(outputs,
861 PoolingRequestOutput)
862
863 + def embed(
864 + self,
865 + prompts: Union[PromptType, Sequence[PromptType]],
866 + /,
867 + *,
868 + use_tqdm: bool = True,
869 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
870 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
871 + ) -> List[EmbeddingRequestOutput]:
872 + """
873 + Generate an embedding vector for each prompt.
874 +
875 + This class automatically batches the given prompts, considering
876 + the memory constraint. For the best performance, put all of your prompts
877 + into a single list and pass it to this method.
878 +
879 + Args:
880 + prompts: The prompts to the LLM. You may pass a sequence of prompts
881 + for batch inference. See :class:`~vllm.inputs.PromptType`
882 + for more details about the format of each prompts.
883 + use_tqdm: Whether to use tqdm to display the progress bar.
884 + lora_request: LoRA request to use for generation, if any.
885 + prompt_adapter_request: Prompt Adapter request to use for
886 + generation, if any.
887 +
888 + Returns:
889 + A list of ``EmbeddingRequestOutput`` objects containing the
890 + embedding vectors in the same order as the input prompts.
891 + """
892 + if self.llm_engine.model_config.task != "embed":
893 + raise ValueError(
894 + "Embedding API is only enabled for `--task embed`")
895 +
896 + items = self.encode(prompts,
897 + use_tqdm=use_tqdm,
898 + lora_request=lora_request,
899 + prompt_adapter_request=prompt_adapter_request)
900 +
901 + return [EmbeddingRequestOutput.from_base(item) for item in items]
902 +
903 + def classify(
904 + self,
905 + prompts: Union[PromptType, Sequence[PromptType]],
906 + /,
907 + *,
908 + use_tqdm: bool = True,
909 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
910 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
911 + ) -> List[ClassificationRequestOutput]:
912 + """
913 + Generate class logits for each prompt.
914 +
915 + This class automatically batches the given prompts, considering
916 + the memory constraint. For the best performance, put all of your prompts
917 + into a single list and pass it to this method.
918 +
919 + Args:
920 + prompts: The prompts to the LLM. You may pass a sequence of prompts
921 + for batch inference. See :class:`~vllm.inputs.PromptType`
922 + for more details about the format of each prompts.
923 + use_tqdm: Whether to use tqdm to display the progress bar.
924 + lora_request: LoRA request to use for generation, if any.
925 + prompt_adapter_request: Prompt Adapter request to use for
926 + generation, if any.
927 +
928 + Returns:
929 + A list of ``ClassificationRequestOutput`` objects containing the
930 + embedding vectors in the same order as the input prompts.
931 + """
932 + if self.llm_engine.model_config.task != "classify":
933 + raise ValueError(
934 + "Classification API is only enabled for `--task classify`")
935 +
936 + items = self.encode(prompts,
937 + use_tqdm=use_tqdm,
938 + lora_request=lora_request,
939 + prompt_adapter_request=prompt_adapter_request)
940 +
941 + return [ClassificationRequestOutput.from_base(item) for item in items]
942 +
943 def score(
944 self,
945 text_1: Union[SingletonPrompt, Sequence[SingletonPrompt]],
946 text_2: Union[SingletonPrompt, Sequence[SingletonPrompt]],
947 /,
948 + *,
949 truncate_prompt_tokens: Optional[int] = None,
950 use_tqdm: bool = True,
951 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
952 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
845 - ) -> List[PoolingRequestOutput]:
846 - """Generates similarity scores for all pairs <text,text_pair>.
953 + ) -> List[ScoringRequestOutput]:
954 + """Generate similarity scores for all pairs ``<text,text_pair>``.
955
848 - The inputs can be 1 -> 1, 1 -> N or N -> N. In the 1 - N case
849 - the text_1 sentence will be replicated N times to pair with the text_2
850 - sentences. The input pairs are used to build a list of prompts for the
956 + The inputs can be ``1 -> 1``, ``1 -> N`` or ``N -> N``.
957 + In the ``1 - N`` case the ``text_1`` sentence will be replicated ``N``
958 + times to pair with the ``text_2`` sentences.
959 + The input pairs are used to build a list of prompts for the
960 cross encoder model. This class automatically batches the prompts,
961 considering the memory constraint. For the best performance, put all
962 of your texts into a single list and pass it to this method.
963
964 Args:
965 text_1: can be a single prompt or a list of prompts, in which
857 - case it has to have the same length as the text_2 list
966 + case it has to have the same length as the ``text_2`` list
967 text_2: The texts to pair with the query to form the input
968 to the LLM. See :class:`~vllm.inputs.PromptType` for
969 more details about the format of each prompts.
@@ -864,7 +973,7 @@ class LLM:
973 generation, if any.
974
975 Returns:
867 - A list of ``PoolingRequestOutput`` objects containing the
976 + A list of ``ScoringRequestOutput`` objects containing the
977 generated scores in the same order as the input prompts.
978 """
979 runner_type = self.llm_engine.model_config.runner_type
@@ -884,6 +993,8 @@ class LLM:
993
994 if not self.llm_engine.model_config.is_cross_encoder:
995 raise ValueError("Your model does not support cross encoding")
996 + if self.llm_engine.model_config.task != "score":
997 + raise ValueError("Score API is only enabled for `--task score`")
998
999 tokenizer = self.llm_engine.get_tokenizer()
1000
@@ -954,8 +1065,10 @@ class LLM:
1065 )
1066
1067 outputs = self._run_engine(use_tqdm=use_tqdm)
957 - return self.engine_class.validate_outputs(outputs,
958 - PoolingRequestOutput)
1068 + items = self.engine_class.validate_outputs(outputs,
1069 + PoolingRequestOutput)
1070 +
1071 + return [ScoringRequestOutput.from_base(item) for item in items]
1072
1073 def start_profile(self) -> None:
1074 self.llm_engine.start_profile()
vllm/entrypoints/openai/protocol.py
+1 -1
@@ -900,7 +900,7 @@ class EmbeddingResponse(OpenAIBaseModel):
900 class ScoreResponseData(OpenAIBaseModel):
901 index: int
902 object: str = "score"
903 - score: Union[List[float], str]
903 + score: float
904
905
906 class ScoreResponse(OpenAIBaseModel):
vllm/entrypoints/openai/serving_embedding.py
+6 -3
@@ -18,14 +18,15 @@ from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
18 ErrorResponse, UsageInfo)
19 from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
20 from vllm.logger import init_logger
21 -from vllm.outputs import PoolingOutput, PoolingRequestOutput
21 +from vllm.outputs import (EmbeddingOutput, EmbeddingRequestOutput,
22 + PoolingRequestOutput)
23 from vllm.utils import merge_async_iterators
24
25 logger = init_logger(__name__)
26
27
28 def _get_embedding(
28 - output: PoolingOutput,
29 + output: EmbeddingOutput,
30 encoding_format: Literal["float", "base64"],
31 ) -> Union[List[float], str]:
32 if encoding_format == "float":
@@ -46,8 +47,10 @@ def request_output_to_embedding_response(
47 data: List[EmbeddingResponseData] = []
48 num_prompt_tokens = 0
49 for idx, final_res in enumerate(final_res_batch):
50 + embedding_res = EmbeddingRequestOutput.from_base(final_res)
51 prompt_token_ids = final_res.prompt_token_ids
50 - embedding = _get_embedding(final_res.outputs, encoding_format)
52 +
53 + embedding = _get_embedding(embedding_res.outputs, encoding_format)
54 embedding_data = EmbeddingResponseData(index=idx, embedding=embedding)
55 data.append(embedding_data)
56
vllm/entrypoints/openai/serving_engine.py
+7 -5
@@ -31,7 +31,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
31 ErrorResponse,
32 LoadLoraAdapterRequest,
33 ModelCard, ModelList,
34 - ModelPermission,
34 + ModelPermission, ScoreRequest,
35 TokenizeChatRequest,
36 TokenizeCompletionRequest,
37 UnloadLoraAdapterRequest)
@@ -73,7 +73,7 @@ class LoRAModulePath:
73
74
75 CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
76 - EmbeddingCompletionRequest,
76 + EmbeddingCompletionRequest, ScoreRequest,
77 TokenizeCompletionRequest]
78
79 ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,
@@ -567,12 +567,14 @@ class OpenAIServing:
567 return None
568
569 @staticmethod
570 - def _base_request_id(raw_request: Request,
570 + def _base_request_id(raw_request: Optional[Request],
571 default: Optional[str] = None) -> Optional[str]:
572 """Pulls the request id to use from a header, if provided"""
573 default = default or random_uuid()
574 - return raw_request.headers.get(
575 - "X-Request-Id", default) if raw_request is not None else default
574 + if raw_request is None:
575 + return default
576 +
577 + return raw_request.headers.get("X-Request-Id", default)
578
579 @staticmethod
580 def _get_decoded_token(logprob: Logprob,
vllm/entrypoints/openai/serving_score.py
+6 -6
@@ -13,7 +13,7 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse, ScoreRequest,
13 from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
14 from vllm.inputs.data import TokensPrompt
15 from vllm.logger import init_logger
16 -from vllm.outputs import PoolingRequestOutput
16 +from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
17 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
18 from vllm.utils import make_async, merge_async_iterators
19
@@ -24,13 +24,13 @@ def request_output_to_score_response(
24 final_res_batch: List[PoolingRequestOutput], request_id: str,
25 created_time: int, model_name: str) -> ScoreResponse:
26 data: List[ScoreResponseData] = []
27 - score = None
27 num_prompt_tokens = 0
28 for idx, final_res in enumerate(final_res_batch):
30 - if final_res is not None:
31 - score = final_res.outputs.embedding
32 - score_data = ScoreResponseData(index=idx, score=score)
33 - data.append(score_data)
29 + classify_res = ScoringRequestOutput.from_base(final_res)
30 +
31 + score_data = ScoreResponseData(index=idx,
32 + score=classify_res.outputs.score)
33 + data.append(score_data)
34
35 usage = UsageInfo(
36 prompt_tokens=num_prompt_tokens,
vllm/model_executor/layers/pooler.py
+196 -92
@@ -1,14 +1,16 @@
1 from enum import IntEnum
2 -from typing import List, Optional
2 +from typing import List, Optional, Union
3
4 import torch
5 import torch.nn as nn
6 +import torch.nn.functional as F
7 from transformers import PretrainedConfig
8 +from typing_extensions import assert_never
9
10 from vllm.config import PoolerConfig
11 from vllm.model_executor.pooling_metadata import (PoolingMetadata,
12 PoolingTensors)
11 -from vllm.sequence import EmbeddingSequenceGroupOutput, PoolerOutput
13 +from vllm.sequence import PoolerOutput, PoolingSequenceGroupOutput
14 from vllm.transformers_utils.config import (
15 get_cross_encoder_activation_function)
16
@@ -22,7 +24,7 @@ class PoolingType(IntEnum):
24 MEAN = 4
25
26
25 -class Pooler(nn.Module):
27 +class SimplePooler(nn.Module):
28 """A layer that pools specific information from hidden states.
29
30 This layer does the following:
@@ -35,21 +37,203 @@ class Pooler(nn.Module):
37 normalize: Whether to normalize the pooled data.
38 """
39
40 + @staticmethod
41 + def from_pooling_type(
42 + pooling_type: PoolingType,
43 + *,
44 + normalize: bool,
45 + softmax: bool,
46 + step_tag_id: Optional[int] = None,
47 + returned_token_ids: Optional[List[int]] = None,
48 + ) -> "SimplePooler":
49 + if pooling_type == PoolingType.LAST:
50 + assert step_tag_id is None and returned_token_ids is None
51 + return LastPool(normalize=normalize, softmax=softmax)
52 + if pooling_type == PoolingType.ALL:
53 + assert step_tag_id is None and returned_token_ids is None
54 + return AllPool(normalize=normalize, softmax=softmax)
55 + if pooling_type == PoolingType.CLS:
56 + assert step_tag_id is None and returned_token_ids is None
57 + return CLSPool(normalize=normalize, softmax=softmax)
58 + if pooling_type == PoolingType.MEAN:
59 + assert step_tag_id is None and returned_token_ids is None
60 + return MeanPool(normalize=normalize, softmax=softmax)
61 + if pooling_type == PoolingType.STEP:
62 + return StepPool(normalize=normalize,
63 + softmax=softmax,
64 + step_tag_id=step_tag_id,
65 + returned_token_ids=returned_token_ids)
66 +
67 + assert_never(pooling_type)
68 +
69 + def __init__(self, *, normalize: bool, softmax: bool) -> None:
70 + super().__init__()
71 +
72 + self.head = PoolerHead(normalize=normalize, softmax=softmax)
73 +
74 + def get_prompt_lens(
75 + self,
76 + hidden_states: torch.Tensor,
77 + pooling_metadata: PoolingMetadata,
78 + ) -> torch.Tensor:
79 + return PoolingTensors.from_pooling_metadata(
80 + pooling_metadata, hidden_states.device).prompt_lens
81 +
82 + def extract_states(
83 + self,
84 + hidden_states: torch.Tensor,
85 + pooling_metadata: PoolingMetadata,
86 + ) -> Union[list[torch.Tensor], torch.Tensor]:
87 + raise NotImplementedError
88 +
89 + def build_output(self, data: torch.Tensor) -> PoolingSequenceGroupOutput:
90 + return PoolingSequenceGroupOutput(data)
91 +
92 + def forward(
93 + self,
94 + hidden_states: torch.Tensor,
95 + pooling_metadata: PoolingMetadata,
96 + ) -> PoolerOutput:
97 + pooled_data = self.extract_states(hidden_states, pooling_metadata)
98 + pooled_data = self.head(pooled_data)
99 + pooled_outputs = [self.build_output(data) for data in pooled_data]
100 + return PoolerOutput(outputs=pooled_outputs)
101 +
102 +
103 +class CLSPool(SimplePooler):
104 +
105 + def extract_states(
106 + self,
107 + hidden_states: torch.Tensor,
108 + pooling_metadata: PoolingMetadata,
109 + ) -> Union[list[torch.Tensor], torch.Tensor]:
110 + prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
111 +
112 + first_token_flat_indices = torch.zeros_like(prompt_lens)
113 + first_token_flat_indices[1:] += torch.cumsum(prompt_lens, dim=0)[:-1]
114 + return hidden_states[first_token_flat_indices]
115 +
116 +
117 +class LastPool(SimplePooler):
118 +
119 + def extract_states(
120 + self,
121 + hidden_states: torch.Tensor,
122 + pooling_metadata: PoolingMetadata,
123 + ) -> Union[list[torch.Tensor], torch.Tensor]:
124 + prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
125 +
126 + last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1
127 + return hidden_states[last_token_flat_indices]
128 +
129 +
130 +class AllPool(SimplePooler):
131 +
132 + def extract_states(
133 + self,
134 + hidden_states: torch.Tensor,
135 + pooling_metadata: PoolingMetadata,
136 + ) -> Union[list[torch.Tensor], torch.Tensor]:
137 + prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
138 +
139 + offset = 0
140 + pooled_data = list[torch.Tensor]()
141 + for prompt_len in prompt_lens:
142 + pooled_data.append(hidden_states[offset:offset + prompt_len])
143 + offset += prompt_len
144 +
145 + return pooled_data
146 +
147 +
148 +class MeanPool(SimplePooler):
149 +
150 + def extract_states(
151 + self,
152 + hidden_states: torch.Tensor,
153 + pooling_metadata: PoolingMetadata,
154 + ) -> Union[list[torch.Tensor], torch.Tensor]:
155 + prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
156 +
157 + cumsum = torch.cumsum(hidden_states, dim=0)
158 + start_indices = torch.cat([
159 + torch.tensor([0], device=hidden_states.device),
160 + torch.cumsum(prompt_lens[:-1], dim=0)
161 + ])
162 + end_indices = torch.cumsum(prompt_lens, dim=0)
163 + return (cumsum[end_indices - 1] - cumsum[start_indices] +
164 + hidden_states[start_indices]) / prompt_lens.unsqueeze(1)
165 +
166 +
167 +class StepPool(SimplePooler):
168 +
169 def __init__(
170 self,
40 - pooling_type: PoolingType,
171 + *,
172 normalize: bool,
173 softmax: bool,
174 step_tag_id: Optional[int] = None,
175 returned_token_ids: Optional[List[int]] = None,
176 ):
177 + super().__init__(normalize=normalize, softmax=softmax)
178 +
179 + self.step_tag_id = step_tag_id
180 + self.returned_token_ids = returned_token_ids
181 +
182 + def extract_states(
183 + self,
184 + hidden_states: torch.Tensor,
185 + pooling_metadata: PoolingMetadata,
186 + ) -> Union[list[torch.Tensor], torch.Tensor]:
187 + prompt_lens = self.get_prompt_lens(hidden_states, pooling_metadata)
188 +
189 + returned_token_ids = self.returned_token_ids
190 + if returned_token_ids is not None and len(returned_token_ids) > 0:
191 + hidden_states = hidden_states[:, returned_token_ids]
192 +
193 + step_tag_id = self.step_tag_id
194 +
195 + offset = 0
196 + pooled_data = list[torch.Tensor]()
197 + for prompt_len, seq_data_i in zip(prompt_lens,
198 + pooling_metadata.seq_data.values()):
199 + pooled_data_i = hidden_states[offset:offset + prompt_len]
200 + if step_tag_id is not None:
201 + token_ids = torch.tensor(seq_data_i.prompt_token_ids)
202 + pooled_data_i = pooled_data_i[token_ids == step_tag_id]
203 +
204 + offset += prompt_len
205 + pooled_data.append(pooled_data_i)
206 +
207 + return pooled_data
208 +
209 +
210 +class PoolerHead(nn.Module):
211 +
212 + def __init__(self, *, normalize: bool, softmax: bool) -> None:
213 super().__init__()
214
48 - self.pooling_type = pooling_type
215 self.normalize = normalize
216 self.softmax = softmax
51 - self.step_tag_id = step_tag_id
52 - self.returned_token_ids = returned_token_ids
217 +
218 + def forward(self, pooled_data: Union[list[torch.Tensor], torch.Tensor]):
219 + if self.normalize:
220 + if isinstance(pooled_data, list):
221 + pooled_data = [
222 + F.normalize(data, p=2, dim=1) for data in pooled_data
223 + ]
224 + else:
225 + pooled_data = F.normalize(pooled_data, p=2, dim=1)
226 +
227 + if self.softmax:
228 + if isinstance(pooled_data, list):
229 + pooled_data = [F.softmax(data, dim=-1) for data in pooled_data]
230 + else:
231 + pooled_data = F.softmax(pooled_data, dim=-1)
232 +
233 + return pooled_data
234 +
235 +
236 +class Pooler(nn.Module):
237
238 @classmethod
239 def from_config_with_defaults(
@@ -60,8 +244,8 @@ class Pooler(nn.Module):
244 softmax: bool,
245 step_tag_id: Optional[int] = None,
246 returned_token_ids: Optional[List[int]] = None,
63 - ) -> "Pooler":
64 - return cls(
247 + ) -> SimplePooler:
248 + return SimplePooler.from_pooling_type(
249 pooling_type=PoolingType[pooler_config.pooling_type]
250 if pooler_config.pooling_type is not None else pooling_type,
251 normalize=pooler_config.normalize
@@ -75,85 +259,6 @@ class Pooler(nn.Module):
259 returned_token_ids,
260 )
261
78 - def forward(
79 - self,
80 - hidden_states: torch.Tensor,
81 - pooling_metadata: PoolingMetadata,
82 - ) -> PoolerOutput:
83 - """Pools specific information from hidden states based on metadata."""
84 -
85 - prompt_lens = PoolingTensors.from_pooling_metadata(
86 - pooling_metadata, hidden_states.device).prompt_lens
87 -
88 - if self.pooling_type is PoolingType.CLS:
89 - first_token_flat_indices = torch.zeros_like(prompt_lens)
90 - first_token_flat_indices[1:] += torch.cumsum(prompt_lens,
91 - dim=0)[:-1]
92 - pooled_data = hidden_states[first_token_flat_indices]
93 - elif self.pooling_type == PoolingType.LAST:
94 - last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1
95 - pooled_data = hidden_states[last_token_flat_indices]
96 - elif self.pooling_type == PoolingType.ALL:
97 - offset = 0
98 - pooled_data = []
99 - for prompt_len in prompt_lens:
100 - pooled_data.append(hidden_states[offset:offset + prompt_len])
101 - offset += prompt_len
102 - elif self.pooling_type == PoolingType.MEAN:
103 - # Calculate mean pooling
104 - cumsum = torch.cumsum(hidden_states, dim=0)
105 - start_indices = torch.cat([
106 - torch.tensor([0], device=hidden_states.device),
107 - torch.cumsum(prompt_lens[:-1], dim=0)
108 - ])
109 - end_indices = torch.cumsum(prompt_lens, dim=0)
110 - pooled_data = (
111 - cumsum[end_indices - 1] - cumsum[start_indices] +
112 - hidden_states[start_indices]) / prompt_lens.unsqueeze(1)
113 - elif self.pooling_type == PoolingType.STEP:
114 - returned_token_ids = self.returned_token_ids
115 - if returned_token_ids is not None and len(returned_token_ids) > 0:
116 - hidden_states = hidden_states[:, returned_token_ids]
117 -
118 - step_tag_id = self.step_tag_id
119 -
120 - offset = 0
121 - pooled_data = []
122 - for prompt_len, seq_data_i in zip(
123 - prompt_lens, pooling_metadata.seq_data.values()):
124 - pooled_data_i = hidden_states[offset:offset + prompt_len]
125 - if step_tag_id is not None:
126 - token_ids = torch.tensor(seq_data_i.prompt_token_ids)
127 - pooled_data_i = pooled_data_i[token_ids == step_tag_id]
128 -
129 - offset += prompt_len
130 - pooled_data.append(pooled_data_i)
131 - else:
132 - raise ValueError(f"Invalid pooling type: {self.pooling_type}")
133 -
134 - if self.normalize:
135 - if isinstance(pooled_data, list):
136 - pooled_data = [
137 - nn.functional.normalize(data, p=2, dim=1)
138 - for data in pooled_data
139 - ]
140 - else:
141 - pooled_data = nn.functional.normalize(pooled_data, p=2, dim=1)
142 -
143 - if self.softmax:
144 - if isinstance(pooled_data, list):
145 - pooled_data = [
146 - nn.functional.softmax(data, dim=-1) for data in pooled_data
147 - ]
148 - else:
149 - pooled_data = nn.functional.softmax(pooled_data, dim=-1)
150 -
151 - pooled_outputs = [
152 - EmbeddingSequenceGroupOutput(data.tolist()) for data in pooled_data
153 - ]
154 -
155 - return PoolerOutput(outputs=pooled_outputs)
156 -
262
263 class CrossEncodingPooler(nn.Module):
264 """A layer that pools specific information from hidden states.
@@ -208,9 +313,8 @@ class CrossEncodingPooler(nn.Module):
313 if self.pooler is not None:
314 # apply classifier once on the full batch if possible
315 pooled_output = self.classifier(pooled_output)
211 - logits = self.default_activation_function(pooled_output)
316
213 - pooled_outputs = [
214 - EmbeddingSequenceGroupOutput(data.tolist()) for data in logits
215 - ]
317 + scores = self.default_activation_function(pooled_output).squeeze(-1)
318 +
319 + pooled_outputs = [PoolingSequenceGroupOutput(data) for data in scores]
320 return PoolerOutput(outputs=pooled_outputs)
vllm/model_executor/models/gritlm.py
+9 -6
@@ -2,19 +2,20 @@ from array import array
2 from typing import List, Optional, Union
3
4 import torch
5 -from torch import nn
5 +import torch.nn as nn
6 from xformers.ops.fmha.attn_bias import BlockDiagonalMask
7
8 from vllm.attention import AttentionMetadata
9 from vllm.attention.backends.xformers import XFormersImpl
10 from vllm.config import ModelConfig, VllmConfig
11 from vllm.logger import init_logger
12 +from vllm.model_executor.layers.pooler import PoolerHead
13 from vllm.model_executor.models.llama import LlamaForCausalLM
14 from vllm.model_executor.pooling_metadata import (PoolingMetadata,
15 PoolingTensors)
16 from vllm.multimodal.utils import cached_get_tokenizer
16 -from vllm.sequence import (EmbeddingSequenceGroupOutput, IntermediateTensors,
17 - PoolerOutput)
17 +from vllm.sequence import (IntermediateTensors, PoolerOutput,
18 + PoolingSequenceGroupOutput)
19
20 logger = init_logger(__name__)
21
@@ -52,6 +53,8 @@ class GritLMPooler(nn.Module):
53 self.embed_pattern_ids = tokens_to_ids(
54 ["▁<", "|", "embed", "|", ">", "<0x0A>"])
55
56 + self.head = PoolerHead(normalize=True, softmax=False)
57 +
58 def _find_array(self, arr: array, target: array, start_idx: int) -> int:
59 """
60 Find the first occurrence of target in arr starting from start_idx.
@@ -75,7 +78,7 @@ class GritLMPooler(nn.Module):
78 return i
79 return -1
80
78 - def _get_instruction_len(self, prompt_token_ids: array) -> bool:
81 + def _get_instruction_len(self, prompt_token_ids: array) -> int:
82 """
83 Get the length of the instruction in the prompt.
84
@@ -168,10 +171,10 @@ class GritLMPooler(nn.Module):
171 mean_embeddings = sum_embeddings / num_non_instruction_tokens.unsqueeze(
172 1)
173
171 - pooled_data = nn.functional.normalize(mean_embeddings, p=2, dim=1)
174 + pooled_data = self.head(mean_embeddings)
175
176 pooled_outputs = [
174 - EmbeddingSequenceGroupOutput(data.tolist()) for data in pooled_data
177 + PoolingSequenceGroupOutput(data) for data in pooled_data
178 ]
179
180 return PoolerOutput(outputs=pooled_outputs)
vllm/outputs.py
+137 -70
@@ -1,9 +1,13 @@
1 import time
2 +import warnings
3 from dataclasses import dataclass
3 -from typing import Dict, List, Optional
4 +from typing import Dict, Generic, List, Optional
5 from typing import Sequence as GenericSequence
6 from typing import Union
7
8 +import torch
9 +from typing_extensions import TypeVar
10 +
11 from vllm.lora.request import LoRARequest
12 from vllm.multimodal.inputs import MultiModalPlaceholderDict
13 from vllm.sampling_params import RequestOutputKind
@@ -57,14 +61,26 @@ class PoolingOutput:
61 """The output data of one pooling output of a request.
62
63 Args:
60 - embedding: The embedding vector, which is a list of floats. The
61 - length of vector depends on the model as listed in the embedding guide.
64 + data: The extracted hidden states.
65 """
63 - embedding: List[float]
66 + data: torch.Tensor
67
68 def __repr__(self) -> str:
66 - return (f"PoolingOutput("
67 - f"embedding={len(self.embedding)})")
69 + return (f"PoolingOutput(data={self.data})")
70 +
71 + def __eq__(self, other: object) -> bool:
72 + return (isinstance(other, self.__class__) and bool(
73 + (self.data == other.data).all()))
74 +
75 + @property
76 + def embedding(self) -> list[float]:
77 + msg = ("`LLM.encode()` now returns raw outputs. "
78 + "To return embeddings, use `LLM.embed()`. "
79 + "To return class probabilities, use `LLM.classify()` "
80 + "and access the `probs` attribute. ")
81 + warnings.warn(msg, DeprecationWarning, stacklevel=2)
82 +
83 + return self.data.tolist()
84
85
86 class RequestOutput:
@@ -316,7 +332,10 @@ class RequestOutput:
332 f"multi_modal_placeholders={self.multi_modal_placeholders})")
333
334
319 -class PoolingRequestOutput:
335 +_O = TypeVar("_O", default=PoolingOutput)
336 +
337 +
338 +class PoolingRequestOutput(Generic[_O]):
339 """
340 The output data of a pooling request to the LLM.
341
@@ -327,24 +346,24 @@ class PoolingRequestOutput:
346 finished (bool): A flag indicating whether the pooling is completed.
347 """
348
330 - def __init__(self, request_id: str, outputs: "PoolingOutput",
349 + def __init__(self, request_id: str, outputs: _O,
350 prompt_token_ids: List[int], finished: bool):
351 self.request_id = request_id
352 self.prompt_token_ids = prompt_token_ids
353 self.finished = finished
354 self.outputs = outputs
355
337 - @classmethod
338 - def from_seq_group(cls,
339 - seq_group: 'SequenceGroup') -> "PoolingRequestOutput":
340 - if seq_group.embeddings is None:
341 - raise ValueError(
342 - "Embeddings are missing in seq_group for EmbeddingRequest.")
343 - output = PoolingOutput(seq_group.embeddings)
356 + @staticmethod
357 + def from_seq_group(seq_group: SequenceGroup) -> "PoolingRequestOutput":
358 + pooled_data = seq_group.pooled_data
359 + assert pooled_data is not None
360 +
361 + output = PoolingOutput(pooled_data)
362 prompt_token_ids = seq_group.prompt_token_ids
363 finished = seq_group.is_finished()
364
347 - return cls(seq_group.request_id, output, prompt_token_ids, finished)
365 + return PoolingRequestOutput(seq_group.request_id, output,
366 + prompt_token_ids, finished)
367
368 def __repr__(self):
369 """
@@ -356,89 +375,137 @@ class PoolingRequestOutput:
375 Returns:
376 str: A string representation of the PoolingRequestOutput instance.
377 """
359 - return (f"PoolingRequestOutput(request_id='{self.request_id}', "
360 - f"outputs={repr(self.outputs)}, "
378 + return (f"{type(self).__name__}(request_id={self.request_id!r}, "
379 + f"outputs={self.outputs!r}, "
380 f"prompt_token_ids={self.prompt_token_ids}, "
381 f"finished={self.finished})")
382
383
384 +class RequestOutputFactory:
385 +
386 + @staticmethod
387 + def create(seq_group: SequenceGroup,
388 + seq_id_to_seq_group: Dict[str, SequenceGroupBase],
389 + use_cache: bool = False):
390 + if seq_group.pooled_data is not None:
391 + return PoolingRequestOutput.from_seq_group(seq_group)
392 + else:
393 + return RequestOutput.from_seq_group(seq_group, use_cache,
394 + seq_id_to_seq_group)
395 +
396 +
397 @dataclass
366 -class ScoreOutput:
367 - """The output data of one completion output of a request.
398 +class EmbeddingOutput:
399 + """The output data of one embedding output of a request.
400
401 Args:
370 - score: The score, which is a list of floats.
371 - index: The correspondent text index of the score.
402 + embedding: The embedding vector, which is a list of floats.
403 + Its length depends on the hidden dimension of the model.
404 """
373 - index: int
374 - score: List[float]
405 + embedding: list[float]
406 +
407 + @staticmethod
408 + def from_base(pooling_output: PoolingOutput):
409 + pooled_data = pooling_output.data
410 + if pooled_data.ndim != 1:
411 + raise ValueError("pooled_data should be a 1-D embedding vector")
412 +
413 + return EmbeddingOutput(pooled_data.tolist())
414 +
415 + @property
416 + def hidden_size(self) -> int:
417 + return len(self.embedding)
418
419 def __repr__(self) -> str:
377 - return (f"ScoreOutput("
378 - f"score={self.score}), "
379 - f"index={self.index})")
420 + return f"EmbeddingOutput(hidden_size={self.hidden_size})"
421
422
382 -class ScoreRequestOutput:
383 - """
384 - The output data of an score request to the LLM.
423 +class EmbeddingRequestOutput(PoolingRequestOutput[EmbeddingOutput]):
424 +
425 + @staticmethod
426 + def from_base(request_output: PoolingRequestOutput):
427 + return EmbeddingRequestOutput(
428 + request_id=request_output.request_id,
429 + outputs=EmbeddingOutput.from_base(request_output.outputs),
430 + prompt_token_ids=request_output.prompt_token_ids,
431 + finished=request_output.finished,
432 + )
433 +
434 +
435 +@dataclass
436 +class ClassificationOutput:
437 + """The output data of one classification output of a request.
438
439 Args:
387 - request_id (str): A unique identifier for the score request.
388 - outputs (score): The embedding results for the given input.
440 + probs: The probability vector, which is a list of floats.
441 + Its length depends on the number of classes.
442 """
443 + probs: list[float]
444
391 - def __init__(self, request_id: str, outputs: "ScoreOutput"):
392 - self.request_id = request_id
393 - self.outputs = outputs
445 + @staticmethod
446 + def from_base(pooling_output: PoolingOutput):
447 + pooled_data = pooling_output.data
448 + if pooled_data.ndim != 1:
449 + raise ValueError("pooled_data should be a 1-D probability vector")
450
395 - def __repr__(self):
396 - """
397 - Returns a string representation of an ScoreRequestOutput instance.
451 + return ClassificationOutput(pooled_data.tolist())
452
399 - The representation includes the request_id and the number of outputs,
400 - providing a quick overview of the embedding request's results.
453 + @property
454 + def num_classes(self) -> int:
455 + return len(self.probs)
456
402 - Returns:
403 - str: A string representation of the ScoreRequestOutput instance.
404 - """
405 - return (f"ScoreRequestOutput(request_id='{self.request_id}', "
406 - f"outputs={repr(self.outputs)}")
457 + def __repr__(self) -> str:
458 + return f"ClassificationOutput(num_classes={self.num_classes})"
459
460
409 -class RequestOutputFactory:
461 +class ClassificationRequestOutput(PoolingRequestOutput[ClassificationOutput]):
462
463 @staticmethod
412 - def create(seq_group: SequenceGroup,
413 - seq_id_to_seq_group: Dict[str, SequenceGroupBase],
414 - use_cache: bool = False):
415 - # Determine the type based on a condition, for example:
416 - if hasattr(seq_group,
417 - 'embeddings') and seq_group.embeddings is not None:
418 - return PoolingRequestOutput.from_seq_group(seq_group)
419 - else:
420 - return RequestOutput.from_seq_group(seq_group, use_cache,
421 - seq_id_to_seq_group)
464 + def from_base(request_output: PoolingRequestOutput):
465 + return ClassificationRequestOutput(
466 + request_id=request_output.request_id,
467 + outputs=ClassificationOutput.from_base(request_output.outputs),
468 + prompt_token_ids=request_output.prompt_token_ids,
469 + finished=request_output.finished,
470 + )
471
472
424 -def __getattr__(name: str):
425 - import warnings
473 +@dataclass
474 +class ScoringOutput:
475 + """The output data of one scoring output of a request.
476
427 - if name == "EmbeddingOutput":
428 - msg = ("EmbeddingOutput has been renamed to PoolingOutput. "
429 - "The original name will be removed in an upcoming version.")
477 + Args:
478 + score: The similarity score, which is a scalar value.
479 + """
480 + score: float
481 +
482 + @staticmethod
483 + def from_base(pooling_output: PoolingOutput):
484 + pooled_data = pooling_output.data
485 + if pooled_data.ndim != 0:
486 + raise ValueError("pooled_data should be a scalar score")
487
431 - warnings.warn(DeprecationWarning(msg), stacklevel=2)
488 + return ScoringOutput(pooled_data.item())
489
433 - return PoolingOutput
490 + def __repr__(self) -> str:
491 + return f"ScoringOutput(score={self.score})"
492
435 - if name == "EmbeddingRequestOutput":
436 - msg = ("EmbeddingRequestOutput has been renamed to "
437 - "PoolingRequestOutput. "
438 - "The original name will be removed in an upcoming version.")
493 + @property
494 + def embedding(self) -> list[float]:
495 + msg = ("`LLM.score()` now returns scalar scores. "
496 + "Please access it via the `score` attribute. ")
497 + warnings.warn(msg, DeprecationWarning, stacklevel=2)
498
440 - warnings.warn(DeprecationWarning(msg), stacklevel=2)
499 + return [self.score]
500
442 - return PoolingRequestOutput
501
444 - raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
502 +class ScoringRequestOutput(PoolingRequestOutput[ScoringOutput]):
503 +
504 + @staticmethod
505 + def from_base(request_output: PoolingRequestOutput):
506 + return ScoringRequestOutput(
507 + request_id=request_output.request_id,
508 + outputs=ScoringOutput.from_base(request_output.outputs),
509 + prompt_token_ids=request_output.prompt_token_ids,
510 + finished=request_output.finished,
511 + )
vllm/sequence.py
+18 -22
@@ -617,10 +617,9 @@ class SequenceGroup:
617 sampling_params: The sampling parameters used to generate the outputs.
618 arrival_time: The arrival time of the request.
619 lora_request: LoRA request.
620 - embeddings: The embeddings vectors of the prompt of the sequence group
621 - for a pooling model.
622 - pooling_params: The pooling parameters used to generate the pooling
620 + pooling_params: The parameters used to generate the pooler
621 for a pooling model.
622 + pooled_data: The extracted hidden states from a pooling model.
623 encoder_seq: Optional, the single encoder sequence. Should be None
624 unless you are working with an encoder/decoder model.
625 trace_headers: OpenTelemetry trace headers.
@@ -635,8 +634,8 @@ class SequenceGroup:
634 arrival_time: float,
635 sampling_params: Optional[SamplingParams] = None,
636 lora_request: Optional[LoRARequest] = None,
638 - embeddings: Optional[List[float]] = None,
637 pooling_params: Optional[PoolingParams] = None,
638 + pooled_data: Optional[torch.Tensor] = None,
639 encoder_seq: Optional[Sequence] = None,
640 trace_headers: Optional[Mapping[str, str]] = None,
641 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -658,8 +657,8 @@ class SequenceGroup:
657 self.lora_request = lora_request
658 self.prompt_logprobs: Optional[PromptLogprobs] = None
659 self.state = SequenceGroupState()
661 - self.embeddings = embeddings
660 self.pooling_params = pooling_params
661 + self.pooled_data = pooled_data
662 self.prompt_adapter_request = prompt_adapter_request
663 self.encoder_seq = encoder_seq
664 self.trace_headers = trace_headers
@@ -1033,8 +1032,8 @@ class CompletionSequenceGroupOutput(
1032 msgspec.Struct,
1033 omit_defaults=True, # type: ignore[call-arg]
1034 array_like=True): # type: ignore[call-arg]
1036 - __metaclass__ = SequenceGroupOutput
1035 """The model output associated with a completion sequence group."""
1036 + __metaclass__ = SequenceGroupOutput
1037 samples: List[SequenceOutput]
1038 # Prompt logprob for each prompt query token.
1039 prompt_logprobs: Optional[PromptLogprobs]
@@ -1050,23 +1049,24 @@ class CompletionSequenceGroupOutput(
1049 and self.prompt_logprobs == other.prompt_logprobs)
1050
1051
1053 -class EmbeddingSequenceGroupOutput(
1052 +class PoolingSequenceGroupOutput(
1053 msgspec.Struct,
1054 omit_defaults=True, # type: ignore[call-arg]
1055 array_like=True, # type: ignore[call-arg]
1056 ):
1058 - """The model output associated with an embedding sequence group."""
1057 + """The model output associated with a pooling sequence group."""
1058 __metaclass__ = SequenceGroupOutput
1060 - embeddings: List[int]
1059 + # Annotated as Any to be compatible with msgspec
1060 + # The actual type is in SequenceGroup.pooled_data
1061 + data: Any
1062
1063 def __repr__(self) -> str:
1063 - return (f"EmbeddingSequenceGroupOutput("
1064 - f"embeddings_shape={len(self.embeddings)})")
1064 + return f"PoolingSequenceGroupOutput(data={self.data}"
1065
1066 def __eq__(self, other: object) -> bool:
1067 - if not isinstance(other, EmbeddingSequenceGroupOutput):
1067 + if not isinstance(other, PoolingSequenceGroupOutput):
1068 raise NotImplementedError()
1069 - return self.embeddings == other.embeddings
1069 + return self.data == other.data
1070
1071
1072 # cannot use msgspec.Struct here because Dynamo does not support it
@@ -1085,7 +1085,7 @@ class IntermediateTensors:
1085 elif isinstance(key, slice):
1086 return self.__class__({k: v[key] for k, v in self.tensors.items()})
1087
1088 - def __setitem__(self, key: str, value):
1088 + def __setitem__(self, key: str, value: torch.Tensor):
1089 self.tensors[key] = value
1090
1091 def __len__(self):
@@ -1103,16 +1103,12 @@ class PoolerOutput(
1103 omit_defaults=True, # type: ignore[call-arg]
1104 array_like=True): # type: ignore[call-arg]
1105 """The output from a pooling operation in the pooling model."""
1106 - outputs: List[EmbeddingSequenceGroupOutput]
1107 -
1108 - # lazy import to avoid circular import
1109 - from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
1110 - spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1106 + outputs: List[PoolingSequenceGroupOutput]
1107
1112 - def __getitem__(self, idx: int) -> EmbeddingSequenceGroupOutput:
1108 + def __getitem__(self, idx: int) -> PoolingSequenceGroupOutput:
1109 return self.outputs[idx]
1110
1115 - def __setitem__(self, idx: int, value):
1111 + def __setitem__(self, idx: int, value: PoolingSequenceGroupOutput):
1112 self.outputs[idx] = value
1113
1114 def __len__(self):
@@ -1385,8 +1381,8 @@ class ParallelSampleSequenceGroup(SequenceGroupBase):
1381 arrival_time=seq_group.arrival_time,
1382 sampling_params=original_params,
1383 lora_request=seq_group.lora_request,
1388 - embeddings=seq_group.embeddings,
1384 pooling_params=seq_group.pooling_params,
1385 + pooled_data=seq_group.pooled_data,
1386 encoder_seq=seq_group.encoder_seq,
1387 trace_headers=seq_group.trace_headers,
1388 prompt_adapter_request=seq_group.prompt_adapter_request,