A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Misc] Rename embedding classes to pooling (#10801)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 1, 2024 at 14:36 UTC
d2f058e76c2a28d2109e163dc1123ead6983943c
25 files changed
+166
-123
examples/offline_inference_embedding.py
+1
-1
@@ -10,7 +10,7 @@ prompts = [
10
11
# Create an LLM.
12
model = LLM(model="intfloat/e5-mistral-7b-instruct", enforce_eager=True)
13
-# Generate embedding. The output is a list of EmbeddingRequestOutputs.
13
+# Generate embedding. The output is a list of PoolingRequestOutputs.
14
outputs = model.encode(prompts)
15
# Print the outputs.
16
for output in outputs:
tests/entrypoints/llm/test_encode.py
+3
-3
@@ -3,7 +3,7 @@ from typing import List
3
4
import pytest
5
6
-from vllm import LLM, EmbeddingRequestOutput, PoolingParams
6
+from vllm import LLM, PoolingParams, PoolingRequestOutput
7
from vllm.distributed import cleanup_dist_env_and_memory
8
9
MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
@@ -43,8 +43,8 @@ def llm():
43
cleanup_dist_env_and_memory()
44
45
46
-def assert_outputs_equal(o1: List[EmbeddingRequestOutput],
47
- o2: List[EmbeddingRequestOutput]):
46
+def assert_outputs_equal(o1: List[PoolingRequestOutput],
47
+ o2: List[PoolingRequestOutput]):
48
assert [o.outputs for o in o1] == [o.outputs for o in o2]
49
50
tests/models/test_registry.py
+2
-2
@@ -3,7 +3,7 @@ import warnings
3
import pytest
4
import torch.cuda
5
6
-from vllm.model_executor.models import (is_embedding_model,
6
+from vllm.model_executor.models import (is_pooling_model,
7
is_text_generation_model,
8
supports_multimodal)
9
from vllm.model_executor.models.adapters import as_embedding_model
@@ -31,7 +31,7 @@ def test_registry_imports(model_arch):
31
32
# All vLLM models should be convertible to an embedding model
33
embed_model = as_embedding_model(model_cls)
34
- assert is_embedding_model(embed_model)
34
+ assert is_pooling_model(embed_model)
35
36
if model_arch in _MULTIMODAL_MODELS:
37
assert supports_multimodal(model_cls)
tests/worker/test_model_input.py
+2
-2
@@ -8,10 +8,10 @@ from vllm.attention.backends.abstract import AttentionBackend
8
from vllm.attention.backends.utils import CommonAttentionState
9
from vllm.model_executor import SamplingMetadata
10
from vllm.model_executor.pooling_metadata import PoolingMetadata
11
-from vllm.worker.embedding_model_runner import (
12
- ModelInputForGPUWithPoolingMetadata)
11
from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
12
from vllm.worker.multi_step_model_runner import StatefulModelInput
13
+from vllm.worker.pooling_model_runner import (
14
+ ModelInputForGPUWithPoolingMetadata)
15
16
17
class MockAttentionBackend(AttentionBackend):
vllm/__init__.py
+27
-4
@@ -7,8 +7,8 @@ from vllm.entrypoints.llm import LLM
7
from vllm.executor.ray_utils import initialize_ray_cluster
8
from vllm.inputs import PromptType, TextPrompt, TokensPrompt
9
from vllm.model_executor.models import ModelRegistry
10
-from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11
- EmbeddingRequestOutput, RequestOutput)
10
+from vllm.outputs import (CompletionOutput, PoolingOutput,
11
+ PoolingRequestOutput, RequestOutput)
12
from vllm.pooling_params import PoolingParams
13
from vllm.sampling_params import SamplingParams
14
@@ -25,8 +25,8 @@ __all__ = [
25
"SamplingParams",
26
"RequestOutput",
27
"CompletionOutput",
28
- "EmbeddingOutput",
29
- "EmbeddingRequestOutput",
28
+ "PoolingOutput",
29
+ "PoolingRequestOutput",
30
"LLMEngine",
31
"EngineArgs",
32
"AsyncLLMEngine",
@@ -34,3 +34,26 @@ __all__ = [
34
"initialize_ray_cluster",
35
"PoolingParams",
36
]
37
+
38
+
39
+def __getattr__(name: str):
40
+ import warnings
41
+
42
+ if name == "EmbeddingOutput":
43
+ msg = ("EmbeddingOutput has been renamed to PoolingOutput. "
44
+ "The original name will be removed in an upcoming version.")
45
+
46
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
47
+
48
+ return PoolingOutput
49
+
50
+ if name == "EmbeddingRequestOutput":
51
+ msg = ("EmbeddingRequestOutput has been renamed to "
52
+ "PoolingRequestOutput. "
53
+ "The original name will be removed in an upcoming version.")
54
+
55
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
56
+
57
+ return PoolingRequestOutput
58
+
59
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/config.py
+1
-1
@@ -359,7 +359,7 @@ class ModelConfig:
359
# NOTE: Listed from highest to lowest priority,
360
# in case the model supports multiple of them
361
"generate": ModelRegistry.is_text_generation_model(architectures),
362
- "embedding": ModelRegistry.is_embedding_model(architectures),
362
+ "embedding": ModelRegistry.is_pooling_model(architectures),
363
}
364
supported_tasks_lst: List[_Task] = [
365
task for task, is_supported in task_support.items() if is_supported
vllm/engine/async_llm_engine.py
+12
-12
@@ -25,7 +25,7 @@ from vllm.lora.request import LoRARequest
25
from vllm.model_executor.guided_decoding import (
26
get_guided_decoding_logits_processor)
27
from vllm.model_executor.layers.sampler import SamplerOutput
28
-from vllm.outputs import EmbeddingRequestOutput, RequestOutput
28
+from vllm.outputs import PoolingRequestOutput, RequestOutput
29
from vllm.pooling_params import PoolingParams
30
from vllm.prompt_adapter.request import PromptAdapterRequest
31
from vllm.sampling_params import SamplingParams
@@ -74,7 +74,7 @@ STOP_ITERATION = Exception() # Sentinel
74
75
76
class AsyncStream:
77
- """A stream of RequestOutputs or EmbeddingRequestOutputs for a request
77
+ """A stream of RequestOutputs or PoolingRequestOutputs for a request
78
that can be iterated over asynchronously via an async generator."""
79
80
def __init__(self, request_id: str, cancel: Callable[[str], None]) -> None:
@@ -83,7 +83,7 @@ class AsyncStream:
83
self._queue: asyncio.Queue = asyncio.Queue()
84
self._finished = False
85
86
- def put(self, item: Union[RequestOutput, EmbeddingRequestOutput,
86
+ def put(self, item: Union[RequestOutput, PoolingRequestOutput,
87
Exception]) -> None:
88
if not self._finished:
89
self._queue.put_nowait(item)
@@ -103,7 +103,7 @@ class AsyncStream:
103
104
async def generator(
105
self
106
- ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
106
+ ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
107
try:
108
while True:
109
result = await self._queue.get()
@@ -154,7 +154,7 @@ class RequestTracker:
154
155
def process_request_output(self,
156
request_output: Union[RequestOutput,
157
- EmbeddingRequestOutput],
157
+ PoolingRequestOutput],
158
*,
159
verbose: bool = False) -> None:
160
"""Process a request output from the engine."""
@@ -265,7 +265,7 @@ class _AsyncLLMEngine(LLMEngine):
265
266
async def step_async(
267
self, virtual_engine: int
268
- ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
268
+ ) -> List[Union[RequestOutput, PoolingRequestOutput]]:
269
"""Performs one decoding iteration and returns newly generated results.
270
The workers are ran asynchronously if possible.
271
@@ -907,7 +907,7 @@ class AsyncLLMEngine(EngineClient):
907
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
908
priority: int = 0,
909
) -> Coroutine[None, None, AsyncGenerator[Union[
910
- RequestOutput, EmbeddingRequestOutput], None]]:
910
+ RequestOutput, PoolingRequestOutput], None]]:
911
...
912
913
@overload
@@ -922,7 +922,7 @@ class AsyncLLMEngine(EngineClient):
922
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
923
priority: int = 0,
924
) -> Coroutine[None, None, AsyncGenerator[Union[
925
- RequestOutput, EmbeddingRequestOutput], None]]:
925
+ RequestOutput, PoolingRequestOutput], None]]:
926
...
927
928
@deprecate_kwargs(
@@ -941,7 +941,7 @@ class AsyncLLMEngine(EngineClient):
941
priority: int = 0,
942
*,
943
inputs: Optional[PromptType] = None, # DEPRECATED
944
- ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
944
+ ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
945
if inputs is not None:
946
prompt = inputs
947
assert prompt is not None and params is not None
@@ -1070,7 +1070,7 @@ class AsyncLLMEngine(EngineClient):
1070
lora_request: Optional[LoRARequest] = None,
1071
trace_headers: Optional[Mapping[str, str]] = None,
1072
priority: int = 0,
1073
- ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
1073
+ ) -> AsyncGenerator[PoolingRequestOutput, None]:
1074
"""Generate outputs for a request from an embedding model.
1075
1076
Generate outputs for a request. This method is a coroutine. It adds the
@@ -1088,7 +1088,7 @@ class AsyncLLMEngine(EngineClient):
1088
Only applicable with priority scheduling.
1089
1090
Yields:
1091
- The output `EmbeddingRequestOutput` objects from the LLMEngine
1091
+ The output `PoolingRequestOutput` objects from the LLMEngine
1092
for the request.
1093
1094
Details:
@@ -1141,7 +1141,7 @@ class AsyncLLMEngine(EngineClient):
1141
trace_headers=trace_headers,
1142
priority=priority,
1143
):
1144
- yield LLMEngine.validate_output(output, EmbeddingRequestOutput)
1144
+ yield LLMEngine.validate_output(output, PoolingRequestOutput)
1145
1146
async def abort(self, request_id: str) -> None:
1147
"""Abort a request.
vllm/engine/llm_engine.py
+4
-4
@@ -40,7 +40,7 @@ from vllm.model_executor.guided_decoding import (
40
get_local_guided_decoding_logits_processor)
41
from vllm.model_executor.layers.sampler import SamplerOutput
42
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
43
-from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
43
+from vllm.outputs import (PoolingRequestOutput, RequestOutput,
44
RequestOutputFactory)
45
from vllm.pooling_params import PoolingParams
46
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -80,7 +80,7 @@ def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
80
81
82
_G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
83
-_O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
83
+_O = TypeVar("_O", RequestOutput, PoolingRequestOutput)
84
85
86
@dataclass
@@ -112,7 +112,7 @@ class SchedulerContext:
112
def __init__(self, multi_step_stream_outputs: bool = False):
113
self.output_queue: Deque[OutputData] = deque()
114
self.request_outputs: List[Union[RequestOutput,
115
- EmbeddingRequestOutput]] = []
115
+ PoolingRequestOutput]] = []
116
self.seq_group_metadata_list: Optional[
117
List[SequenceGroupMetadata]] = None
118
self.scheduler_outputs: Optional[SchedulerOutputs] = None
@@ -1314,7 +1314,7 @@ class LLMEngine:
1314
else:
1315
seq.append_token_id(sample.output_token, sample.logprobs)
1316
1317
- def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1317
+ def step(self) -> List[Union[RequestOutput, PoolingRequestOutput]]:
1318
"""Performs one decoding iteration and returns newly generated results.
1319
1320
.. figure:: https://i.imgur.com/sv2HssD.png
vllm/engine/multiprocessing/client.py
+7
-7
@@ -35,7 +35,7 @@ from vllm.inputs.preprocess import InputPreprocessor
35
from vllm.logger import init_logger
36
from vllm.lora.request import LoRARequest
37
from vllm.model_executor.layers.sampler import SamplerOutput
38
-from vllm.outputs import EmbeddingRequestOutput, RequestOutput
38
+from vllm.outputs import PoolingRequestOutput, RequestOutput
39
from vllm.prompt_adapter.request import PromptAdapterRequest
40
from vllm.sampling_params import SamplingParams
41
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
@@ -495,7 +495,7 @@ class MQLLMEngineClient(EngineClient):
495
lora_request: Optional[LoRARequest] = None,
496
trace_headers: Optional[Mapping[str, str]] = None,
497
priority: int = 0,
498
- ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
498
+ ) -> AsyncGenerator[PoolingRequestOutput, None]:
499
...
500
501
@overload
@@ -507,7 +507,7 @@ class MQLLMEngineClient(EngineClient):
507
lora_request: Optional[LoRARequest] = None,
508
trace_headers: Optional[Mapping[str, str]] = None,
509
priority: int = 0,
510
- ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
510
+ ) -> AsyncGenerator[PoolingRequestOutput, None]:
511
...
512
513
@deprecate_kwargs(
@@ -524,7 +524,7 @@ class MQLLMEngineClient(EngineClient):
524
priority: int = 0,
525
*,
526
inputs: Optional[PromptType] = None # DEPRECATED
527
- ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
527
+ ) -> AsyncGenerator[PoolingRequestOutput, None]:
528
"""Generate outputs for a request from an embedding model.
529
530
Generate outputs for a request. This method is a coroutine. It adds the
@@ -540,7 +540,7 @@ class MQLLMEngineClient(EngineClient):
540
trace_headers: OpenTelemetry trace headers.
541
542
Yields:
543
- The output `EmbeddingRequestOutput` objects from the LLMEngine
543
+ The output `PoolingRequestOutput` objects from the LLMEngine
544
for the request.
545
"""
546
if inputs is not None:
@@ -549,7 +549,7 @@ class MQLLMEngineClient(EngineClient):
549
and request_id is not None)
550
551
return cast(
552
- AsyncGenerator[EmbeddingRequestOutput, None],
552
+ AsyncGenerator[PoolingRequestOutput, None],
553
self._process_request(prompt,
554
pooling_params,
555
request_id,
@@ -567,7 +567,7 @@ class MQLLMEngineClient(EngineClient):
567
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
568
priority: int = 0,
569
) -> Union[AsyncGenerator[RequestOutput, None], AsyncGenerator[
570
- EmbeddingRequestOutput, None]]:
570
+ PoolingRequestOutput, None]]:
571
"""Send an RPCGenerateRequest to the RPCServer and stream responses."""
572
573
# If already dead, error out.
vllm/engine/protocol.py
+2
-3
@@ -11,8 +11,7 @@ from vllm.inputs.preprocess import InputPreprocessor
11
from vllm.logger import init_logger
12
from vllm.lora.request import LoRARequest
13
from vllm.model_executor.layers.sampler import SamplerOutput
14
-from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput,
15
- RequestOutput)
14
+from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
15
from vllm.pooling_params import PoolingParams
16
from vllm.prompt_adapter.request import PromptAdapterRequest
17
from vllm.sampling_params import BeamSearchParams, SamplingParams
@@ -209,7 +208,7 @@ class EngineClient(ABC):
208
lora_request: Optional[LoRARequest] = None,
209
trace_headers: Optional[Mapping[str, str]] = None,
210
priority: int = 0,
212
- ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
211
+ ) -> AsyncGenerator[PoolingRequestOutput, None]:
212
"""Generate outputs for a request from an embedding model."""
213
...
214
vllm/entrypoints/llm.py
+15
-15
@@ -26,7 +26,7 @@ from vllm.logger import init_logger
26
from vllm.lora.request import LoRARequest
27
from vllm.model_executor.guided_decoding.guided_fields import (
28
GuidedDecodingRequest, LLMGuidedOptions)
29
-from vllm.outputs import EmbeddingRequestOutput, RequestOutput
29
+from vllm.outputs import PoolingRequestOutput, RequestOutput
30
from vllm.pooling_params import PoolingParams
31
from vllm.prompt_adapter.request import PromptAdapterRequest
32
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
@@ -679,7 +679,7 @@ class LLM:
679
prompt_token_ids: Optional[List[int]] = None,
680
use_tqdm: bool = True,
681
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
682
- ) -> List[EmbeddingRequestOutput]:
682
+ ) -> List[PoolingRequestOutput]:
683
...
684
685
@overload # LEGACY: multi (prompt + optional token ids)
@@ -691,7 +691,7 @@ class LLM:
691
prompt_token_ids: Optional[List[List[int]]] = None,
692
use_tqdm: bool = True,
693
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
694
- ) -> List[EmbeddingRequestOutput]:
694
+ ) -> List[PoolingRequestOutput]:
695
...
696
697
@overload # LEGACY: single (token ids + optional prompt)
@@ -704,7 +704,7 @@ class LLM:
704
prompt_token_ids: List[int],
705
use_tqdm: bool = True,
706
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
707
- ) -> List[EmbeddingRequestOutput]:
707
+ ) -> List[PoolingRequestOutput]:
708
...
709
710
@overload # LEGACY: multi (token ids + optional prompt)
@@ -717,7 +717,7 @@ class LLM:
717
prompt_token_ids: List[List[int]],
718
use_tqdm: bool = True,
719
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
720
- ) -> List[EmbeddingRequestOutput]:
720
+ ) -> List[PoolingRequestOutput]:
721
...
722
723
@overload # LEGACY: single or multi token ids [pos-only]
@@ -728,7 +728,7 @@ class LLM:
728
prompt_token_ids: Union[List[int], List[List[int]]],
729
use_tqdm: bool = True,
730
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
731
- ) -> List[EmbeddingRequestOutput]:
731
+ ) -> List[PoolingRequestOutput]:
732
...
733
734
@overload
@@ -741,7 +741,7 @@ class LLM:
741
Sequence[PoolingParams]]] = None,
742
use_tqdm: bool = True,
743
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
744
- ) -> List[EmbeddingRequestOutput]:
744
+ ) -> List[PoolingRequestOutput]:
745
...
746
747
@deprecate_kwargs(
@@ -759,7 +759,7 @@ class LLM:
759
use_tqdm: bool = True,
760
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
761
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
762
- ) -> List[EmbeddingRequestOutput]:
762
+ ) -> List[PoolingRequestOutput]:
763
"""Generates the completions for the input prompts.
764
765
This class automatically batches the given prompts, considering
@@ -778,7 +778,7 @@ class LLM:
778
generation, if any.
779
780
Returns:
781
- A list of ``EmbeddingRequestOutput`` objects containing the
781
+ A list of ``PoolingRequestOutput`` objects containing the
782
generated embeddings in the same order as the input prompts.
783
784
Note:
@@ -821,7 +821,7 @@ class LLM:
821
822
outputs = self._run_engine(use_tqdm=use_tqdm)
823
return self.engine_class.validate_outputs(outputs,
824
- EmbeddingRequestOutput)
824
+ PoolingRequestOutput)
825
826
def score(
827
self,
@@ -832,7 +832,7 @@ class LLM:
832
use_tqdm: bool = True,
833
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
834
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
835
- ) -> List[EmbeddingRequestOutput]:
835
+ ) -> List[PoolingRequestOutput]:
836
"""Generates similarity scores for all pairs <text,text_pair>.
837
838
The inputs can be 1 -> 1, 1 -> N or N -> N. In the 1 - N case
@@ -854,7 +854,7 @@ class LLM:
854
generation, if any.
855
856
Returns:
857
- A list of ``EmbeddingRequestOutput`` objects containing the
857
+ A list of ``PoolingRequestOutput`` objects containing the
858
generated scores in the same order as the input prompts.
859
"""
860
task = self.llm_engine.model_config.task
@@ -943,7 +943,7 @@ class LLM:
943
944
outputs = self._run_engine(use_tqdm=use_tqdm)
945
return self.engine_class.validate_outputs(outputs,
946
- EmbeddingRequestOutput)
946
+ PoolingRequestOutput)
947
948
def start_profile(self) -> None:
949
self.llm_engine.start_profile()
@@ -1085,7 +1085,7 @@ class LLM:
1085
1086
def _run_engine(
1087
self, *, use_tqdm: bool
1088
- ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1088
+ ) -> List[Union[RequestOutput, PoolingRequestOutput]]:
1089
# Initialize tqdm.
1090
if use_tqdm:
1091
num_requests = self.llm_engine.get_num_unfinished_requests()
@@ -1098,7 +1098,7 @@ class LLM:
1098
)
1099
1100
# Run the engine.
1101
- outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = []
1101
+ outputs: List[Union[RequestOutput, PoolingRequestOutput]] = []
1102
total_in_toks = 0
1103
total_out_toks = 0
1104
while self.llm_engine.has_unfinished_requests():
vllm/entrypoints/openai/serving_embedding.py
+6
-6
@@ -18,14 +18,14 @@ from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
18
ErrorResponse, UsageInfo)
19
from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
20
from vllm.logger import init_logger
21
-from vllm.outputs import EmbeddingOutput, EmbeddingRequestOutput
21
+from vllm.outputs import PoolingOutput, PoolingRequestOutput
22
from vllm.utils import merge_async_iterators, random_uuid
23
24
logger = init_logger(__name__)
25
26
27
def _get_embedding(
28
- output: EmbeddingOutput,
28
+ output: PoolingOutput,
29
encoding_format: Literal["float", "base64"],
30
) -> Union[List[float], str]:
31
if encoding_format == "float":
@@ -40,7 +40,7 @@ def _get_embedding(
40
41
42
def request_output_to_embedding_response(
43
- final_res_batch: List[EmbeddingRequestOutput], request_id: str,
43
+ final_res_batch: List[PoolingRequestOutput], request_id: str,
44
created_time: int, model_name: str,
45
encoding_format: Literal["float", "base64"]) -> EmbeddingResponse:
46
data: List[EmbeddingResponseData] = []
@@ -169,7 +169,7 @@ class OpenAIServingEmbedding(OpenAIServing):
169
return self.create_error_response(str(e))
170
171
# Schedule the request and get the result generator.
172
- generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
172
+ generators: List[AsyncGenerator[PoolingRequestOutput, None]] = []
173
try:
174
pooling_params = request.to_pooling_params()
175
@@ -207,7 +207,7 @@ class OpenAIServingEmbedding(OpenAIServing):
207
num_prompts = len(engine_prompts)
208
209
# Non-streaming response
210
- final_res_batch: List[Optional[EmbeddingRequestOutput]]
210
+ final_res_batch: List[Optional[PoolingRequestOutput]]
211
final_res_batch = [None] * num_prompts
212
try:
213
async for i, res in result_generator:
@@ -215,7 +215,7 @@ class OpenAIServingEmbedding(OpenAIServing):
215
216
assert all(final_res is not None for final_res in final_res_batch)
217
218
- final_res_batch_checked = cast(List[EmbeddingRequestOutput],
218
+ final_res_batch_checked = cast(List[PoolingRequestOutput],
219
final_res_batch)
220
221
response = request_output_to_embedding_response(
vllm/entrypoints/openai/serving_score.py
+5
-5
@@ -13,7 +13,7 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse, ScoreRequest,
13
from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
14
from vllm.inputs.data import TokensPrompt
15
from vllm.logger import init_logger
16
-from vllm.outputs import EmbeddingRequestOutput
16
+from vllm.outputs import PoolingRequestOutput
17
from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
18
from vllm.utils import make_async, merge_async_iterators, random_uuid
19
@@ -21,7 +21,7 @@ logger = init_logger(__name__)
21
22
23
def request_output_to_score_response(
24
- final_res_batch: List[EmbeddingRequestOutput], request_id: str,
24
+ final_res_batch: List[PoolingRequestOutput], request_id: str,
25
created_time: int, model_name: str) -> ScoreResponse:
26
data: List[ScoreResponseData] = []
27
score = None
@@ -133,7 +133,7 @@ class OpenAIServingScores(OpenAIServing):
133
return self.create_error_response(str(e))
134
135
# Schedule the request and get the result generator.
136
- generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
136
+ generators: List[AsyncGenerator[PoolingRequestOutput, None]] = []
137
138
input_pairs = make_pairs(request.text_1, request.text_2)
139
@@ -194,7 +194,7 @@ class OpenAIServingScores(OpenAIServing):
194
num_prompts = len(engine_prompts)
195
196
# Non-streaming response
197
- final_res_batch: List[Optional[EmbeddingRequestOutput]]
197
+ final_res_batch: List[Optional[PoolingRequestOutput]]
198
final_res_batch = [None] * num_prompts
199
200
try:
@@ -203,7 +203,7 @@ class OpenAIServingScores(OpenAIServing):
203
204
assert all(final_res is not None for final_res in final_res_batch)
205
206
- final_res_batch_checked = cast(List[EmbeddingRequestOutput],
206
+ final_res_batch_checked = cast(List[PoolingRequestOutput],
207
final_res_batch)
208
209
response = request_output_to_score_response(
vllm/model_executor/models/__init__.py
+5
-6
@@ -1,15 +1,14 @@
1
from .interfaces import (HasInnerState, SupportsLoRA, SupportsMultiModal,
2
SupportsPP, has_inner_state, supports_lora,
3
supports_multimodal, supports_pp)
4
-from .interfaces_base import (VllmModelForEmbedding,
5
- VllmModelForTextGeneration, is_embedding_model,
6
- is_text_generation_model)
4
+from .interfaces_base import (VllmModelForPooling, VllmModelForTextGeneration,
5
+ is_pooling_model, is_text_generation_model)
6
from .registry import ModelRegistry
7
8
__all__ = [
9
"ModelRegistry",
11
- "VllmModelForEmbedding",
12
- "is_embedding_model",
10
+ "VllmModelForPooling",
11
+ "is_pooling_model",
12
"VllmModelForTextGeneration",
13
"is_text_generation_model",
14
"HasInnerState",
@@ -20,4 +19,4 @@ __all__ = [
19
"supports_multimodal",
20
"SupportsPP",
21
"supports_pp",
23
-]
\ No newline at end of file
22
+]
vllm/model_executor/models/adapters.py
+3
-3
@@ -4,7 +4,7 @@ from typing import Any, TypeVar
4
import torch
5
import torch.nn as nn
6
7
-from .interfaces_base import VllmModelForEmbedding, is_embedding_model
7
+from .interfaces_base import VllmModelForPooling, is_pooling_model
8
9
_T = TypeVar("_T", bound=type[nn.Module])
10
@@ -12,7 +12,7 @@ _T = TypeVar("_T", bound=type[nn.Module])
12
def as_embedding_model(cls: _T) -> _T:
13
"""Subclass an existing vLLM model to support embeddings."""
14
# Avoid modifying existing embedding models
15
- if is_embedding_model(cls):
15
+ if is_pooling_model(cls):
16
return cls
17
18
# Lazy import
@@ -23,7 +23,7 @@ def as_embedding_model(cls: _T) -> _T:
23
24
from .utils import AutoWeightsLoader, WeightsMapper
25
26
- class ModelForEmbedding(cls, VllmModelForEmbedding):
26
+ class ModelForEmbedding(cls, VllmModelForPooling):
27
28
def __init__(
29
self,
vllm/model_executor/models/interfaces.py
+2
-2
@@ -7,7 +7,7 @@ from typing_extensions import TypeIs, TypeVar
7
from vllm.logger import init_logger
8
from vllm.utils import supports_kw
9
10
-from .interfaces_base import is_embedding_model
10
+from .interfaces_base import is_pooling_model
11
12
if TYPE_CHECKING:
13
from vllm.attention import AttentionMetadata
@@ -389,4 +389,4 @@ def _supports_cross_encoding(
389
def supports_cross_encoding(
390
model: Union[Type[object], object],
391
) -> Union[TypeIs[Type[SupportsCrossEncoding]], TypeIs[SupportsCrossEncoding]]:
392
- return is_embedding_model(model) and _supports_cross_encoding(model)
392
+ return is_pooling_model(model) and _supports_cross_encoding(model)
vllm/model_executor/models/interfaces_base.py
+7
-8
@@ -141,7 +141,7 @@ def is_text_generation_model(
141
142
143
@runtime_checkable
144
-class VllmModelForEmbedding(VllmModel[C_co, T], Protocol[C_co, T]):
144
+class VllmModelForPooling(VllmModel[C_co, T], Protocol[C_co, T]):
145
146
def pooler(
147
self,
@@ -153,23 +153,22 @@ class VllmModelForEmbedding(VllmModel[C_co, T], Protocol[C_co, T]):
153
154
155
@overload
156
-def is_embedding_model(
157
- model: Type[object]) -> TypeIs[Type[VllmModelForEmbedding]]:
156
+def is_pooling_model(model: Type[object]) -> TypeIs[Type[VllmModelForPooling]]:
157
...
158
159
160
@overload
162
-def is_embedding_model(model: object) -> TypeIs[VllmModelForEmbedding]:
161
+def is_pooling_model(model: object) -> TypeIs[VllmModelForPooling]:
162
...
163
164
166
-def is_embedding_model(
165
+def is_pooling_model(
166
model: Union[Type[object], object],
168
-) -> Union[TypeIs[Type[VllmModelForEmbedding]], TypeIs[VllmModelForEmbedding]]:
167
+) -> Union[TypeIs[Type[VllmModelForPooling]], TypeIs[VllmModelForPooling]]:
168
if not is_vllm_model(model):
169
return False
170
171
if isinstance(model, type):
173
- return isinstance(model, VllmModelForEmbedding)
172
+ return isinstance(model, VllmModelForPooling)
173
175
- return isinstance(model, VllmModelForEmbedding)
174
+ return isinstance(model, VllmModelForPooling)
vllm/model_executor/models/registry.py
+8
-8
@@ -24,7 +24,7 @@ from .adapters import as_embedding_model
24
from .interfaces import (has_inner_state, is_attention_free,
25
supports_cross_encoding, supports_multimodal,
26
supports_pp)
27
-from .interfaces_base import is_embedding_model, is_text_generation_model
27
+from .interfaces_base import is_pooling_model, is_text_generation_model
28
29
logger = init_logger(__name__)
30
@@ -211,7 +211,7 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
211
class _ModelInfo:
212
architecture: str
213
is_text_generation_model: bool
214
- is_embedding_model: bool
214
+ is_pooling_model: bool
215
supports_cross_encoding: bool
216
supports_multimodal: bool
217
supports_pp: bool
@@ -220,19 +220,19 @@ class _ModelInfo:
220
221
@staticmethod
222
def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
223
- is_embedding_model_ = is_embedding_model(model)
224
- if not is_embedding_model_:
223
+ is_pooling_model_ = is_pooling_model(model)
224
+ if not is_pooling_model_:
225
try:
226
as_embedding_model(model)
227
except Exception:
228
pass
229
else:
230
- is_embedding_model_ = True
230
+ is_pooling_model_ = True
231
232
return _ModelInfo(
233
architecture=model.__name__,
234
is_text_generation_model=is_text_generation_model(model),
235
- is_embedding_model=is_embedding_model_,
235
+ is_pooling_model=is_pooling_model_,
236
supports_cross_encoding=supports_cross_encoding(model),
237
supports_multimodal=supports_multimodal(model),
238
supports_pp=supports_pp(model),
@@ -441,12 +441,12 @@ class _ModelRegistry:
441
model_cls, _ = self.inspect_model_cls(architectures)
442
return model_cls.is_text_generation_model
443
444
- def is_embedding_model(
444
+ def is_pooling_model(
445
self,
446
architectures: Union[str, List[str]],
447
) -> bool:
448
model_cls, _ = self.inspect_model_cls(architectures)
449
- return model_cls.is_embedding_model
449
+ return model_cls.is_pooling_model
450
451
def is_cross_encoder_model(
452
self,
vllm/outputs.py
+39
-16
@@ -53,8 +53,8 @@ class CompletionOutput:
53
54
55
@dataclass
56
-class EmbeddingOutput:
57
- """The output data of one completion output of a request.
56
+class PoolingOutput:
57
+ """The output data of one pooling output of a request.
58
59
Args:
60
embedding: The embedding vector, which is a list of floats. The
@@ -63,7 +63,7 @@ class EmbeddingOutput:
63
embedding: List[float]
64
65
def __repr__(self) -> str:
66
- return (f"EmbeddingOutput("
66
+ return (f"PoolingOutput("
67
f"embedding={len(self.embedding)})")
68
69
@@ -316,18 +316,18 @@ class RequestOutput:
316
f"multi_modal_placeholders={self.multi_modal_placeholders})")
317
318
319
-class EmbeddingRequestOutput:
319
+class PoolingRequestOutput:
320
"""
321
- The output data of an embedding request to the LLM.
321
+ The output data of a pooling request to the LLM.
322
323
Args:
324
- request_id (str): A unique identifier for the embedding request.
325
- outputs (EmbeddingOutput): The embedding results for the given input.
324
+ request_id (str): A unique identifier for the pooling request.
325
+ outputs (PoolingOutput): The pooling results for the given input.
326
prompt_token_ids (List[int]): A list of token IDs used in the prompt.
327
- finished (bool): A flag indicating whether the embedding is completed.
327
+ finished (bool): A flag indicating whether the pooling is completed.
328
"""
329
330
- def __init__(self, request_id: str, outputs: "EmbeddingOutput",
330
+ def __init__(self, request_id: str, outputs: "PoolingOutput",
331
prompt_token_ids: List[int], finished: bool):
332
self.request_id = request_id
333
self.prompt_token_ids = prompt_token_ids
@@ -336,11 +336,11 @@ class EmbeddingRequestOutput:
336
337
@classmethod
338
def from_seq_group(cls,
339
- seq_group: 'SequenceGroup') -> "EmbeddingRequestOutput":
339
+ seq_group: 'SequenceGroup') -> "PoolingRequestOutput":
340
if seq_group.embeddings is None:
341
raise ValueError(
342
"Embeddings are missing in seq_group for EmbeddingRequest.")
343
- output = EmbeddingOutput(seq_group.embeddings)
343
+ output = PoolingOutput(seq_group.embeddings)
344
prompt_token_ids = seq_group.prompt_token_ids
345
finished = seq_group.is_finished()
346
@@ -348,15 +348,15 @@ class EmbeddingRequestOutput:
348
349
def __repr__(self):
350
"""
351
- Returns a string representation of an EmbeddingRequestOutput instance.
351
+ Returns a string representation of an PoolingRequestOutput instance.
352
353
The representation includes the request_id and the number of outputs,
354
- providing a quick overview of the embedding request's results.
354
+ providing a quick overview of the pooling request's results.
355
356
Returns:
357
- str: A string representation of the EmbeddingRequestOutput instance.
357
+ str: A string representation of the PoolingRequestOutput instance.
358
"""
359
- return (f"EmbeddingRequestOutput(request_id='{self.request_id}', "
359
+ return (f"PoolingRequestOutput(request_id='{self.request_id}', "
360
f"outputs={repr(self.outputs)}, "
361
f"prompt_token_ids={self.prompt_token_ids}, "
362
f"finished={self.finished})")
@@ -415,7 +415,30 @@ class RequestOutputFactory:
415
# Determine the type based on a condition, for example:
416
if hasattr(seq_group,
417
'embeddings') and seq_group.embeddings is not None:
418
- return EmbeddingRequestOutput.from_seq_group(seq_group)
418
+ return PoolingRequestOutput.from_seq_group(seq_group)
419
else:
420
return RequestOutput.from_seq_group(seq_group, use_cache,
421
seq_id_to_seq_group)
422
+
423
+
424
+def __getattr__(name: str):
425
+ import warnings
426
+
427
+ if name == "EmbeddingOutput":
428
+ msg = ("EmbeddingOutput has been renamed to PoolingOutput. "
429
+ "The original name will be removed in an upcoming version.")
430
+
431
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
432
+
433
+ return PoolingOutput
434
+
435
+ if name == "EmbeddingRequestOutput":
436
+ msg = ("EmbeddingRequestOutput has been renamed to "
437
+ "PoolingRequestOutput. "
438
+ "The original name will be removed in an upcoming version.")
439
+
440
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
441
+
442
+ return PoolingRequestOutput
443
+
444
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/v1/engine/async_llm.py
+2
-2
@@ -9,7 +9,7 @@ from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
9
from vllm.inputs.preprocess import InputPreprocessor
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
-from vllm.outputs import EmbeddingRequestOutput, RequestOutput
12
+from vllm.outputs import PoolingRequestOutput, RequestOutput
13
from vllm.pooling_params import PoolingParams
14
from vllm.prompt_adapter.request import PromptAdapterRequest
15
from vllm.sampling_params import SamplingParams
@@ -133,7 +133,7 @@ class AsyncLLM(EngineClient):
133
trace_headers: Optional[Mapping[str, str]] = None,
134
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
135
priority: int = 0,
136
- ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
136
+ ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
137
"""Add new request to the AsyncLLM."""
138
139
if self.detokenizer.is_request_active(request_id):
vllm/v1/engine/async_stream.py
+4
-4
@@ -1,11 +1,11 @@
1
import asyncio
2
from typing import Any, AsyncGenerator, Callable, Optional, Type, Union
3
4
-from vllm.outputs import EmbeddingRequestOutput, RequestOutput
4
+from vllm.outputs import PoolingRequestOutput, RequestOutput
5
6
7
class AsyncStream:
8
- """A stream of RequestOutputs or EmbeddingRequestOutputs for a request
8
+ """A stream of RequestOutputs or PoolingRequestOutputs for a request
9
that can be iterated over asynchronously via an async generator."""
10
11
STOP_ITERATION = Exception() # Sentinel
@@ -16,7 +16,7 @@ class AsyncStream:
16
self._queue: asyncio.Queue = asyncio.Queue()
17
self._finished = False
18
19
- def put(self, item: Union[RequestOutput, EmbeddingRequestOutput,
19
+ def put(self, item: Union[RequestOutput, PoolingRequestOutput,
20
Exception]) -> None:
21
if not self._finished:
22
self._queue.put_nowait(item)
@@ -32,7 +32,7 @@ class AsyncStream:
32
33
async def generator(
34
self
35
- ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
35
+ ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
36
finished = False
37
try:
38
while True:
vllm/worker/cpu_pooling_model_runner.py
renamed
+2
-2
@@ -16,12 +16,12 @@ from vllm.worker.cpu_model_runner import (CPUModelRunnerBase, ModelInputForCPU,
16
@dataclasses.dataclass(frozen=True)
17
class ModelInputForCPUWithPoolingMetadata(ModelInputForCPU):
18
"""
19
- Used by the CPUEmbeddingModelRunner.
19
+ Used by the CPUPoolingModelRunner.
20
"""
21
pooling_metadata: Optional["PoolingMetadata"] = None
22
23
24
-class CPUEmbeddingModelRunner(
24
+class CPUPoolingModelRunner(
25
CPUModelRunnerBase[ModelInputForCPUWithPoolingMetadata]):
26
_model_input_cls: Type[ModelInputForCPUWithPoolingMetadata] = (
27
ModelInputForCPUWithPoolingMetadata)
vllm/worker/cpu_worker.py
+2
-2
@@ -14,9 +14,9 @@ from vllm.logger import init_logger
14
from vllm.model_executor import set_random_seed
15
from vllm.sequence import ExecuteModelRequest
16
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
17
-from vllm.worker.cpu_embedding_model_runner import CPUEmbeddingModelRunner
17
from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner
18
from vllm.worker.cpu_model_runner import CPUModelRunner, CPUModelRunnerBase
19
+from vllm.worker.cpu_pooling_model_runner import CPUPoolingModelRunner
20
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
21
LoraNotSupportedWorkerBase, WorkerBase,
22
WorkerInput)
@@ -164,7 +164,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
164
else {"return_hidden_states": True}
165
ModelRunnerClass: Type[CPUModelRunnerBase] = CPUModelRunner
166
if self.model_config.task == "embedding":
167
- ModelRunnerClass = CPUEmbeddingModelRunner
167
+ ModelRunnerClass = CPUPoolingModelRunner
168
elif self.model_config.is_encoder_decoder:
169
ModelRunnerClass = CPUEncoderDecoderModelRunner
170
self.model_runner: CPUModelRunnerBase = ModelRunnerClass(
vllm/worker/pooling_model_runner.py
renamed
+3
-3
@@ -21,12 +21,12 @@ logger = init_logger(__name__)
21
@dataclasses.dataclass(frozen=True)
22
class ModelInputForGPUWithPoolingMetadata(ModelInputForGPU):
23
"""
24
- Used by the EmbeddingModelRunner.
24
+ Used by the PoolingModelRunner.
25
"""
26
pooling_metadata: Optional["PoolingMetadata"] = None
27
28
29
-class EmbeddingModelRunner(
29
+class PoolingModelRunner(
30
GPUModelRunnerBase[ModelInputForGPUWithPoolingMetadata]):
31
_model_input_cls: Type[ModelInputForGPUWithPoolingMetadata] = (
32
ModelInputForGPUWithPoolingMetadata)
@@ -52,7 +52,7 @@ class EmbeddingModelRunner(
52
) -> Optional[Union[List[PoolerOutput], IntermediateTensors]]:
53
if num_steps > 1:
54
raise ValueError(
55
- "EmbeddingModelRunner does not support multi-step execution.")
55
+ "PoolingModelRunner does not support multi-step execution.")
56
57
if self.lora_config:
58
assert model_input.lora_requests is not None
vllm/worker/worker.py
+2
-2
@@ -22,9 +22,9 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
22
from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
23
SequenceGroupMetadata, SequenceGroupMetadataDelta)
24
from vllm.worker.cache_engine import CacheEngine
25
-from vllm.worker.embedding_model_runner import EmbeddingModelRunner
25
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
26
from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
27
+from vllm.worker.pooling_model_runner import PoolingModelRunner
28
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, WorkerBase,
29
WorkerInput)
30
@@ -75,7 +75,7 @@ class Worker(LocalOrDistributedWorkerBase):
75
76
ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
77
if model_config.task == "embedding":
78
- ModelRunnerClass = EmbeddingModelRunner
78
+ ModelRunnerClass = PoolingModelRunner
79
elif self.model_config.is_encoder_decoder:
80
ModelRunnerClass = EncoderDecoderModelRunner
81
self.model_runner: GPUModelRunnerBase = ModelRunnerClass(