@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d2f058e76c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Misc] Rename embedding classes to pooling (#10801)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 1, 2024 at 14:36 UTC d2f058e76c2a28d2109e163dc1123ead6983943c
25 files changed +166 -123
examples/offline_inference_embedding.py
+1 -1
@@ -10,7 +10,7 @@ prompts = [
10
11 # Create an LLM.
12 model = LLM(model="intfloat/e5-mistral-7b-instruct", enforce_eager=True)
13 -# Generate embedding. The output is a list of EmbeddingRequestOutputs.
13 +# Generate embedding. The output is a list of PoolingRequestOutputs.
14 outputs = model.encode(prompts)
15 # Print the outputs.
16 for output in outputs:
tests/entrypoints/llm/test_encode.py
+3 -3
@@ -3,7 +3,7 @@ from typing import List
3
4 import pytest
5
6 -from vllm import LLM, EmbeddingRequestOutput, PoolingParams
6 +from vllm import LLM, PoolingParams, PoolingRequestOutput
7 from vllm.distributed import cleanup_dist_env_and_memory
8
9 MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
@@ -43,8 +43,8 @@ def llm():
43 cleanup_dist_env_and_memory()
44
45
46 -def assert_outputs_equal(o1: List[EmbeddingRequestOutput],
47 - o2: List[EmbeddingRequestOutput]):
46 +def assert_outputs_equal(o1: List[PoolingRequestOutput],
47 + o2: List[PoolingRequestOutput]):
48 assert [o.outputs for o in o1] == [o.outputs for o in o2]
49
50
tests/models/test_registry.py
+2 -2
@@ -3,7 +3,7 @@ import warnings
3 import pytest
4 import torch.cuda
5
6 -from vllm.model_executor.models import (is_embedding_model,
6 +from vllm.model_executor.models import (is_pooling_model,
7 is_text_generation_model,
8 supports_multimodal)
9 from vllm.model_executor.models.adapters import as_embedding_model
@@ -31,7 +31,7 @@ def test_registry_imports(model_arch):
31
32 # All vLLM models should be convertible to an embedding model
33 embed_model = as_embedding_model(model_cls)
34 - assert is_embedding_model(embed_model)
34 + assert is_pooling_model(embed_model)
35
36 if model_arch in _MULTIMODAL_MODELS:
37 assert supports_multimodal(model_cls)
tests/worker/test_model_input.py
+2 -2
@@ -8,10 +8,10 @@ from vllm.attention.backends.abstract import AttentionBackend
8 from vllm.attention.backends.utils import CommonAttentionState
9 from vllm.model_executor import SamplingMetadata
10 from vllm.model_executor.pooling_metadata import PoolingMetadata
11 -from vllm.worker.embedding_model_runner import (
12 - ModelInputForGPUWithPoolingMetadata)
11 from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
12 from vllm.worker.multi_step_model_runner import StatefulModelInput
13 +from vllm.worker.pooling_model_runner import (
14 + ModelInputForGPUWithPoolingMetadata)
15
16
17 class MockAttentionBackend(AttentionBackend):
vllm/__init__.py
+27 -4
@@ -7,8 +7,8 @@ from vllm.entrypoints.llm import LLM
7 from vllm.executor.ray_utils import initialize_ray_cluster
8 from vllm.inputs import PromptType, TextPrompt, TokensPrompt
9 from vllm.model_executor.models import ModelRegistry
10 -from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11 - EmbeddingRequestOutput, RequestOutput)
10 +from vllm.outputs import (CompletionOutput, PoolingOutput,
11 + PoolingRequestOutput, RequestOutput)
12 from vllm.pooling_params import PoolingParams
13 from vllm.sampling_params import SamplingParams
14
@@ -25,8 +25,8 @@ __all__ = [
25 "SamplingParams",
26 "RequestOutput",
27 "CompletionOutput",
28 - "EmbeddingOutput",
29 - "EmbeddingRequestOutput",
28 + "PoolingOutput",
29 + "PoolingRequestOutput",
30 "LLMEngine",
31 "EngineArgs",
32 "AsyncLLMEngine",
@@ -34,3 +34,26 @@ __all__ = [
34 "initialize_ray_cluster",
35 "PoolingParams",
36 ]
37 +
38 +
39 +def __getattr__(name: str):
40 + import warnings
41 +
42 + if name == "EmbeddingOutput":
43 + msg = ("EmbeddingOutput has been renamed to PoolingOutput. "
44 + "The original name will be removed in an upcoming version.")
45 +
46 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
47 +
48 + return PoolingOutput
49 +
50 + if name == "EmbeddingRequestOutput":
51 + msg = ("EmbeddingRequestOutput has been renamed to "
52 + "PoolingRequestOutput. "
53 + "The original name will be removed in an upcoming version.")
54 +
55 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
56 +
57 + return PoolingRequestOutput
58 +
59 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/config.py
+1 -1
@@ -359,7 +359,7 @@ class ModelConfig:
359 # NOTE: Listed from highest to lowest priority,
360 # in case the model supports multiple of them
361 "generate": ModelRegistry.is_text_generation_model(architectures),
362 - "embedding": ModelRegistry.is_embedding_model(architectures),
362 + "embedding": ModelRegistry.is_pooling_model(architectures),
363 }
364 supported_tasks_lst: List[_Task] = [
365 task for task, is_supported in task_support.items() if is_supported
vllm/engine/async_llm_engine.py
+12 -12
@@ -25,7 +25,7 @@ from vllm.lora.request import LoRARequest
25 from vllm.model_executor.guided_decoding import (
26 get_guided_decoding_logits_processor)
27 from vllm.model_executor.layers.sampler import SamplerOutput
28 -from vllm.outputs import EmbeddingRequestOutput, RequestOutput
28 +from vllm.outputs import PoolingRequestOutput, RequestOutput
29 from vllm.pooling_params import PoolingParams
30 from vllm.prompt_adapter.request import PromptAdapterRequest
31 from vllm.sampling_params import SamplingParams
@@ -74,7 +74,7 @@ STOP_ITERATION = Exception() # Sentinel
74
75
76 class AsyncStream:
77 - """A stream of RequestOutputs or EmbeddingRequestOutputs for a request
77 + """A stream of RequestOutputs or PoolingRequestOutputs for a request
78 that can be iterated over asynchronously via an async generator."""
79
80 def __init__(self, request_id: str, cancel: Callable[[str], None]) -> None:
@@ -83,7 +83,7 @@ class AsyncStream:
83 self._queue: asyncio.Queue = asyncio.Queue()
84 self._finished = False
85
86 - def put(self, item: Union[RequestOutput, EmbeddingRequestOutput,
86 + def put(self, item: Union[RequestOutput, PoolingRequestOutput,
87 Exception]) -> None:
88 if not self._finished:
89 self._queue.put_nowait(item)
@@ -103,7 +103,7 @@ class AsyncStream:
103
104 async def generator(
105 self
106 - ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
106 + ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
107 try:
108 while True:
109 result = await self._queue.get()
@@ -154,7 +154,7 @@ class RequestTracker:
154
155 def process_request_output(self,
156 request_output: Union[RequestOutput,
157 - EmbeddingRequestOutput],
157 + PoolingRequestOutput],
158 *,
159 verbose: bool = False) -> None:
160 """Process a request output from the engine."""
@@ -265,7 +265,7 @@ class _AsyncLLMEngine(LLMEngine):
265
266 async def step_async(
267 self, virtual_engine: int
268 - ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
268 + ) -> List[Union[RequestOutput, PoolingRequestOutput]]:
269 """Performs one decoding iteration and returns newly generated results.
270 The workers are ran asynchronously if possible.
271
@@ -907,7 +907,7 @@ class AsyncLLMEngine(EngineClient):
907 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
908 priority: int = 0,
909 ) -> Coroutine[None, None, AsyncGenerator[Union[
910 - RequestOutput, EmbeddingRequestOutput], None]]:
910 + RequestOutput, PoolingRequestOutput], None]]:
911 ...
912
913 @overload
@@ -922,7 +922,7 @@ class AsyncLLMEngine(EngineClient):
922 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
923 priority: int = 0,
924 ) -> Coroutine[None, None, AsyncGenerator[Union[
925 - RequestOutput, EmbeddingRequestOutput], None]]:
925 + RequestOutput, PoolingRequestOutput], None]]:
926 ...
927
928 @deprecate_kwargs(
@@ -941,7 +941,7 @@ class AsyncLLMEngine(EngineClient):
941 priority: int = 0,
942 *,
943 inputs: Optional[PromptType] = None, # DEPRECATED
944 - ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
944 + ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
945 if inputs is not None:
946 prompt = inputs
947 assert prompt is not None and params is not None
@@ -1070,7 +1070,7 @@ class AsyncLLMEngine(EngineClient):
1070 lora_request: Optional[LoRARequest] = None,
1071 trace_headers: Optional[Mapping[str, str]] = None,
1072 priority: int = 0,
1073 - ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
1073 + ) -> AsyncGenerator[PoolingRequestOutput, None]:
1074 """Generate outputs for a request from an embedding model.
1075
1076 Generate outputs for a request. This method is a coroutine. It adds the
@@ -1088,7 +1088,7 @@ class AsyncLLMEngine(EngineClient):
1088 Only applicable with priority scheduling.
1089
1090 Yields:
1091 - The output `EmbeddingRequestOutput` objects from the LLMEngine
1091 + The output `PoolingRequestOutput` objects from the LLMEngine
1092 for the request.
1093
1094 Details:
@@ -1141,7 +1141,7 @@ class AsyncLLMEngine(EngineClient):
1141 trace_headers=trace_headers,
1142 priority=priority,
1143 ):
1144 - yield LLMEngine.validate_output(output, EmbeddingRequestOutput)
1144 + yield LLMEngine.validate_output(output, PoolingRequestOutput)
1145
1146 async def abort(self, request_id: str) -> None:
1147 """Abort a request.
vllm/engine/llm_engine.py
+4 -4
@@ -40,7 +40,7 @@ from vllm.model_executor.guided_decoding import (
40 get_local_guided_decoding_logits_processor)
41 from vllm.model_executor.layers.sampler import SamplerOutput
42 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
43 -from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
43 +from vllm.outputs import (PoolingRequestOutput, RequestOutput,
44 RequestOutputFactory)
45 from vllm.pooling_params import PoolingParams
46 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -80,7 +80,7 @@ def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
80
81
82 _G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
83 -_O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
83 +_O = TypeVar("_O", RequestOutput, PoolingRequestOutput)
84
85
86 @dataclass
@@ -112,7 +112,7 @@ class SchedulerContext:
112 def __init__(self, multi_step_stream_outputs: bool = False):
113 self.output_queue: Deque[OutputData] = deque()
114 self.request_outputs: List[Union[RequestOutput,
115 - EmbeddingRequestOutput]] = []
115 + PoolingRequestOutput]] = []
116 self.seq_group_metadata_list: Optional[
117 List[SequenceGroupMetadata]] = None
118 self.scheduler_outputs: Optional[SchedulerOutputs] = None
@@ -1314,7 +1314,7 @@ class LLMEngine:
1314 else:
1315 seq.append_token_id(sample.output_token, sample.logprobs)
1316
1317 - def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1317 + def step(self) -> List[Union[RequestOutput, PoolingRequestOutput]]:
1318 """Performs one decoding iteration and returns newly generated results.
1319
1320 .. figure:: https://i.imgur.com/sv2HssD.png
vllm/engine/multiprocessing/client.py
+7 -7
@@ -35,7 +35,7 @@ from vllm.inputs.preprocess import InputPreprocessor
35 from vllm.logger import init_logger
36 from vllm.lora.request import LoRARequest
37 from vllm.model_executor.layers.sampler import SamplerOutput
38 -from vllm.outputs import EmbeddingRequestOutput, RequestOutput
38 +from vllm.outputs import PoolingRequestOutput, RequestOutput
39 from vllm.prompt_adapter.request import PromptAdapterRequest
40 from vllm.sampling_params import SamplingParams
41 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
@@ -495,7 +495,7 @@ class MQLLMEngineClient(EngineClient):
495 lora_request: Optional[LoRARequest] = None,
496 trace_headers: Optional[Mapping[str, str]] = None,
497 priority: int = 0,
498 - ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
498 + ) -> AsyncGenerator[PoolingRequestOutput, None]:
499 ...
500
501 @overload
@@ -507,7 +507,7 @@ class MQLLMEngineClient(EngineClient):
507 lora_request: Optional[LoRARequest] = None,
508 trace_headers: Optional[Mapping[str, str]] = None,
509 priority: int = 0,
510 - ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
510 + ) -> AsyncGenerator[PoolingRequestOutput, None]:
511 ...
512
513 @deprecate_kwargs(
@@ -524,7 +524,7 @@ class MQLLMEngineClient(EngineClient):
524 priority: int = 0,
525 *,
526 inputs: Optional[PromptType] = None # DEPRECATED
527 - ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
527 + ) -> AsyncGenerator[PoolingRequestOutput, None]:
528 """Generate outputs for a request from an embedding model.
529
530 Generate outputs for a request. This method is a coroutine. It adds the
@@ -540,7 +540,7 @@ class MQLLMEngineClient(EngineClient):
540 trace_headers: OpenTelemetry trace headers.
541
542 Yields:
543 - The output `EmbeddingRequestOutput` objects from the LLMEngine
543 + The output `PoolingRequestOutput` objects from the LLMEngine
544 for the request.
545 """
546 if inputs is not None:
@@ -549,7 +549,7 @@ class MQLLMEngineClient(EngineClient):
549 and request_id is not None)
550
551 return cast(
552 - AsyncGenerator[EmbeddingRequestOutput, None],
552 + AsyncGenerator[PoolingRequestOutput, None],
553 self._process_request(prompt,
554 pooling_params,
555 request_id,
@@ -567,7 +567,7 @@ class MQLLMEngineClient(EngineClient):
567 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
568 priority: int = 0,
569 ) -> Union[AsyncGenerator[RequestOutput, None], AsyncGenerator[
570 - EmbeddingRequestOutput, None]]:
570 + PoolingRequestOutput, None]]:
571 """Send an RPCGenerateRequest to the RPCServer and stream responses."""
572
573 # If already dead, error out.
vllm/engine/protocol.py
+2 -3
@@ -11,8 +11,7 @@ from vllm.inputs.preprocess import InputPreprocessor
11 from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
13 from vllm.model_executor.layers.sampler import SamplerOutput
14 -from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput,
15 - RequestOutput)
14 +from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
15 from vllm.pooling_params import PoolingParams
16 from vllm.prompt_adapter.request import PromptAdapterRequest
17 from vllm.sampling_params import BeamSearchParams, SamplingParams
@@ -209,7 +208,7 @@ class EngineClient(ABC):
208 lora_request: Optional[LoRARequest] = None,
209 trace_headers: Optional[Mapping[str, str]] = None,
210 priority: int = 0,
212 - ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
211 + ) -> AsyncGenerator[PoolingRequestOutput, None]:
212 """Generate outputs for a request from an embedding model."""
213 ...
214
vllm/entrypoints/llm.py
+15 -15
@@ -26,7 +26,7 @@ from vllm.logger import init_logger
26 from vllm.lora.request import LoRARequest
27 from vllm.model_executor.guided_decoding.guided_fields import (
28 GuidedDecodingRequest, LLMGuidedOptions)
29 -from vllm.outputs import EmbeddingRequestOutput, RequestOutput
29 +from vllm.outputs import PoolingRequestOutput, RequestOutput
30 from vllm.pooling_params import PoolingParams
31 from vllm.prompt_adapter.request import PromptAdapterRequest
32 from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
@@ -679,7 +679,7 @@ class LLM:
679 prompt_token_ids: Optional[List[int]] = None,
680 use_tqdm: bool = True,
681 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
682 - ) -> List[EmbeddingRequestOutput]:
682 + ) -> List[PoolingRequestOutput]:
683 ...
684
685 @overload # LEGACY: multi (prompt + optional token ids)
@@ -691,7 +691,7 @@ class LLM:
691 prompt_token_ids: Optional[List[List[int]]] = None,
692 use_tqdm: bool = True,
693 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
694 - ) -> List[EmbeddingRequestOutput]:
694 + ) -> List[PoolingRequestOutput]:
695 ...
696
697 @overload # LEGACY: single (token ids + optional prompt)
@@ -704,7 +704,7 @@ class LLM:
704 prompt_token_ids: List[int],
705 use_tqdm: bool = True,
706 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
707 - ) -> List[EmbeddingRequestOutput]:
707 + ) -> List[PoolingRequestOutput]:
708 ...
709
710 @overload # LEGACY: multi (token ids + optional prompt)
@@ -717,7 +717,7 @@ class LLM:
717 prompt_token_ids: List[List[int]],
718 use_tqdm: bool = True,
719 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
720 - ) -> List[EmbeddingRequestOutput]:
720 + ) -> List[PoolingRequestOutput]:
721 ...
722
723 @overload # LEGACY: single or multi token ids [pos-only]
@@ -728,7 +728,7 @@ class LLM:
728 prompt_token_ids: Union[List[int], List[List[int]]],
729 use_tqdm: bool = True,
730 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
731 - ) -> List[EmbeddingRequestOutput]:
731 + ) -> List[PoolingRequestOutput]:
732 ...
733
734 @overload
@@ -741,7 +741,7 @@ class LLM:
741 Sequence[PoolingParams]]] = None,
742 use_tqdm: bool = True,
743 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
744 - ) -> List[EmbeddingRequestOutput]:
744 + ) -> List[PoolingRequestOutput]:
745 ...
746
747 @deprecate_kwargs(
@@ -759,7 +759,7 @@ class LLM:
759 use_tqdm: bool = True,
760 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
761 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
762 - ) -> List[EmbeddingRequestOutput]:
762 + ) -> List[PoolingRequestOutput]:
763 """Generates the completions for the input prompts.
764
765 This class automatically batches the given prompts, considering
@@ -778,7 +778,7 @@ class LLM:
778 generation, if any.
779
780 Returns:
781 - A list of ``EmbeddingRequestOutput`` objects containing the
781 + A list of ``PoolingRequestOutput`` objects containing the
782 generated embeddings in the same order as the input prompts.
783
784 Note:
@@ -821,7 +821,7 @@ class LLM:
821
822 outputs = self._run_engine(use_tqdm=use_tqdm)
823 return self.engine_class.validate_outputs(outputs,
824 - EmbeddingRequestOutput)
824 + PoolingRequestOutput)
825
826 def score(
827 self,
@@ -832,7 +832,7 @@ class LLM:
832 use_tqdm: bool = True,
833 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
834 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
835 - ) -> List[EmbeddingRequestOutput]:
835 + ) -> List[PoolingRequestOutput]:
836 """Generates similarity scores for all pairs <text,text_pair>.
837
838 The inputs can be 1 -> 1, 1 -> N or N -> N. In the 1 - N case
@@ -854,7 +854,7 @@ class LLM:
854 generation, if any.
855
856 Returns:
857 - A list of ``EmbeddingRequestOutput`` objects containing the
857 + A list of ``PoolingRequestOutput`` objects containing the
858 generated scores in the same order as the input prompts.
859 """
860 task = self.llm_engine.model_config.task
@@ -943,7 +943,7 @@ class LLM:
943
944 outputs = self._run_engine(use_tqdm=use_tqdm)
945 return self.engine_class.validate_outputs(outputs,
946 - EmbeddingRequestOutput)
946 + PoolingRequestOutput)
947
948 def start_profile(self) -> None:
949 self.llm_engine.start_profile()
@@ -1085,7 +1085,7 @@ class LLM:
1085
1086 def _run_engine(
1087 self, *, use_tqdm: bool
1088 - ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1088 + ) -> List[Union[RequestOutput, PoolingRequestOutput]]:
1089 # Initialize tqdm.
1090 if use_tqdm:
1091 num_requests = self.llm_engine.get_num_unfinished_requests()
@@ -1098,7 +1098,7 @@ class LLM:
1098 )
1099
1100 # Run the engine.
1101 - outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = []
1101 + outputs: List[Union[RequestOutput, PoolingRequestOutput]] = []
1102 total_in_toks = 0
1103 total_out_toks = 0
1104 while self.llm_engine.has_unfinished_requests():
vllm/entrypoints/openai/serving_embedding.py
+6 -6
@@ -18,14 +18,14 @@ from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
18 ErrorResponse, UsageInfo)
19 from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
20 from vllm.logger import init_logger
21 -from vllm.outputs import EmbeddingOutput, EmbeddingRequestOutput
21 +from vllm.outputs import PoolingOutput, PoolingRequestOutput
22 from vllm.utils import merge_async_iterators, random_uuid
23
24 logger = init_logger(__name__)
25
26
27 def _get_embedding(
28 - output: EmbeddingOutput,
28 + output: PoolingOutput,
29 encoding_format: Literal["float", "base64"],
30 ) -> Union[List[float], str]:
31 if encoding_format == "float":
@@ -40,7 +40,7 @@ def _get_embedding(
40
41
42 def request_output_to_embedding_response(
43 - final_res_batch: List[EmbeddingRequestOutput], request_id: str,
43 + final_res_batch: List[PoolingRequestOutput], request_id: str,
44 created_time: int, model_name: str,
45 encoding_format: Literal["float", "base64"]) -> EmbeddingResponse:
46 data: List[EmbeddingResponseData] = []
@@ -169,7 +169,7 @@ class OpenAIServingEmbedding(OpenAIServing):
169 return self.create_error_response(str(e))
170
171 # Schedule the request and get the result generator.
172 - generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
172 + generators: List[AsyncGenerator[PoolingRequestOutput, None]] = []
173 try:
174 pooling_params = request.to_pooling_params()
175
@@ -207,7 +207,7 @@ class OpenAIServingEmbedding(OpenAIServing):
207 num_prompts = len(engine_prompts)
208
209 # Non-streaming response
210 - final_res_batch: List[Optional[EmbeddingRequestOutput]]
210 + final_res_batch: List[Optional[PoolingRequestOutput]]
211 final_res_batch = [None] * num_prompts
212 try:
213 async for i, res in result_generator:
@@ -215,7 +215,7 @@ class OpenAIServingEmbedding(OpenAIServing):
215
216 assert all(final_res is not None for final_res in final_res_batch)
217
218 - final_res_batch_checked = cast(List[EmbeddingRequestOutput],
218 + final_res_batch_checked = cast(List[PoolingRequestOutput],
219 final_res_batch)
220
221 response = request_output_to_embedding_response(
vllm/entrypoints/openai/serving_score.py
+5 -5
@@ -13,7 +13,7 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse, ScoreRequest,
13 from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
14 from vllm.inputs.data import TokensPrompt
15 from vllm.logger import init_logger
16 -from vllm.outputs import EmbeddingRequestOutput
16 +from vllm.outputs import PoolingRequestOutput
17 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
18 from vllm.utils import make_async, merge_async_iterators, random_uuid
19
@@ -21,7 +21,7 @@ logger = init_logger(__name__)
21
22
23 def request_output_to_score_response(
24 - final_res_batch: List[EmbeddingRequestOutput], request_id: str,
24 + final_res_batch: List[PoolingRequestOutput], request_id: str,
25 created_time: int, model_name: str) -> ScoreResponse:
26 data: List[ScoreResponseData] = []
27 score = None
@@ -133,7 +133,7 @@ class OpenAIServingScores(OpenAIServing):
133 return self.create_error_response(str(e))
134
135 # Schedule the request and get the result generator.
136 - generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
136 + generators: List[AsyncGenerator[PoolingRequestOutput, None]] = []
137
138 input_pairs = make_pairs(request.text_1, request.text_2)
139
@@ -194,7 +194,7 @@ class OpenAIServingScores(OpenAIServing):
194 num_prompts = len(engine_prompts)
195
196 # Non-streaming response
197 - final_res_batch: List[Optional[EmbeddingRequestOutput]]
197 + final_res_batch: List[Optional[PoolingRequestOutput]]
198 final_res_batch = [None] * num_prompts
199
200 try:
@@ -203,7 +203,7 @@ class OpenAIServingScores(OpenAIServing):
203
204 assert all(final_res is not None for final_res in final_res_batch)
205
206 - final_res_batch_checked = cast(List[EmbeddingRequestOutput],
206 + final_res_batch_checked = cast(List[PoolingRequestOutput],
207 final_res_batch)
208
209 response = request_output_to_score_response(
vllm/model_executor/models/__init__.py
+5 -6
@@ -1,15 +1,14 @@
1 from .interfaces import (HasInnerState, SupportsLoRA, SupportsMultiModal,
2 SupportsPP, has_inner_state, supports_lora,
3 supports_multimodal, supports_pp)
4 -from .interfaces_base import (VllmModelForEmbedding,
5 - VllmModelForTextGeneration, is_embedding_model,
6 - is_text_generation_model)
4 +from .interfaces_base import (VllmModelForPooling, VllmModelForTextGeneration,
5 + is_pooling_model, is_text_generation_model)
6 from .registry import ModelRegistry
7
8 __all__ = [
9 "ModelRegistry",
11 - "VllmModelForEmbedding",
12 - "is_embedding_model",
10 + "VllmModelForPooling",
11 + "is_pooling_model",
12 "VllmModelForTextGeneration",
13 "is_text_generation_model",
14 "HasInnerState",
@@ -20,4 +19,4 @@ __all__ = [
19 "supports_multimodal",
20 "SupportsPP",
21 "supports_pp",
23 -]
\ No newline at end of file
22 +]
vllm/model_executor/models/adapters.py
+3 -3
@@ -4,7 +4,7 @@ from typing import Any, TypeVar
4 import torch
5 import torch.nn as nn
6
7 -from .interfaces_base import VllmModelForEmbedding, is_embedding_model
7 +from .interfaces_base import VllmModelForPooling, is_pooling_model
8
9 _T = TypeVar("_T", bound=type[nn.Module])
10
@@ -12,7 +12,7 @@ _T = TypeVar("_T", bound=type[nn.Module])
12 def as_embedding_model(cls: _T) -> _T:
13 """Subclass an existing vLLM model to support embeddings."""
14 # Avoid modifying existing embedding models
15 - if is_embedding_model(cls):
15 + if is_pooling_model(cls):
16 return cls
17
18 # Lazy import
@@ -23,7 +23,7 @@ def as_embedding_model(cls: _T) -> _T:
23
24 from .utils import AutoWeightsLoader, WeightsMapper
25
26 - class ModelForEmbedding(cls, VllmModelForEmbedding):
26 + class ModelForEmbedding(cls, VllmModelForPooling):
27
28 def __init__(
29 self,
vllm/model_executor/models/interfaces.py
+2 -2
@@ -7,7 +7,7 @@ from typing_extensions import TypeIs, TypeVar
7 from vllm.logger import init_logger
8 from vllm.utils import supports_kw
9
10 -from .interfaces_base import is_embedding_model
10 +from .interfaces_base import is_pooling_model
11
12 if TYPE_CHECKING:
13 from vllm.attention import AttentionMetadata
@@ -389,4 +389,4 @@ def _supports_cross_encoding(
389 def supports_cross_encoding(
390 model: Union[Type[object], object],
391 ) -> Union[TypeIs[Type[SupportsCrossEncoding]], TypeIs[SupportsCrossEncoding]]:
392 - return is_embedding_model(model) and _supports_cross_encoding(model)
392 + return is_pooling_model(model) and _supports_cross_encoding(model)
vllm/model_executor/models/interfaces_base.py
+7 -8
@@ -141,7 +141,7 @@ def is_text_generation_model(
141
142
143 @runtime_checkable
144 -class VllmModelForEmbedding(VllmModel[C_co, T], Protocol[C_co, T]):
144 +class VllmModelForPooling(VllmModel[C_co, T], Protocol[C_co, T]):
145
146 def pooler(
147 self,
@@ -153,23 +153,22 @@ class VllmModelForEmbedding(VllmModel[C_co, T], Protocol[C_co, T]):
153
154
155 @overload
156 -def is_embedding_model(
157 - model: Type[object]) -> TypeIs[Type[VllmModelForEmbedding]]:
156 +def is_pooling_model(model: Type[object]) -> TypeIs[Type[VllmModelForPooling]]:
157 ...
158
159
160 @overload
162 -def is_embedding_model(model: object) -> TypeIs[VllmModelForEmbedding]:
161 +def is_pooling_model(model: object) -> TypeIs[VllmModelForPooling]:
162 ...
163
164
166 -def is_embedding_model(
165 +def is_pooling_model(
166 model: Union[Type[object], object],
168 -) -> Union[TypeIs[Type[VllmModelForEmbedding]], TypeIs[VllmModelForEmbedding]]:
167 +) -> Union[TypeIs[Type[VllmModelForPooling]], TypeIs[VllmModelForPooling]]:
168 if not is_vllm_model(model):
169 return False
170
171 if isinstance(model, type):
173 - return isinstance(model, VllmModelForEmbedding)
172 + return isinstance(model, VllmModelForPooling)
173
175 - return isinstance(model, VllmModelForEmbedding)
174 + return isinstance(model, VllmModelForPooling)
vllm/model_executor/models/registry.py
+8 -8
@@ -24,7 +24,7 @@ from .adapters import as_embedding_model
24 from .interfaces import (has_inner_state, is_attention_free,
25 supports_cross_encoding, supports_multimodal,
26 supports_pp)
27 -from .interfaces_base import is_embedding_model, is_text_generation_model
27 +from .interfaces_base import is_pooling_model, is_text_generation_model
28
29 logger = init_logger(__name__)
30
@@ -211,7 +211,7 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
211 class _ModelInfo:
212 architecture: str
213 is_text_generation_model: bool
214 - is_embedding_model: bool
214 + is_pooling_model: bool
215 supports_cross_encoding: bool
216 supports_multimodal: bool
217 supports_pp: bool
@@ -220,19 +220,19 @@ class _ModelInfo:
220
221 @staticmethod
222 def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
223 - is_embedding_model_ = is_embedding_model(model)
224 - if not is_embedding_model_:
223 + is_pooling_model_ = is_pooling_model(model)
224 + if not is_pooling_model_:
225 try:
226 as_embedding_model(model)
227 except Exception:
228 pass
229 else:
230 - is_embedding_model_ = True
230 + is_pooling_model_ = True
231
232 return _ModelInfo(
233 architecture=model.__name__,
234 is_text_generation_model=is_text_generation_model(model),
235 - is_embedding_model=is_embedding_model_,
235 + is_pooling_model=is_pooling_model_,
236 supports_cross_encoding=supports_cross_encoding(model),
237 supports_multimodal=supports_multimodal(model),
238 supports_pp=supports_pp(model),
@@ -441,12 +441,12 @@ class _ModelRegistry:
441 model_cls, _ = self.inspect_model_cls(architectures)
442 return model_cls.is_text_generation_model
443
444 - def is_embedding_model(
444 + def is_pooling_model(
445 self,
446 architectures: Union[str, List[str]],
447 ) -> bool:
448 model_cls, _ = self.inspect_model_cls(architectures)
449 - return model_cls.is_embedding_model
449 + return model_cls.is_pooling_model
450
451 def is_cross_encoder_model(
452 self,
vllm/outputs.py
+39 -16
@@ -53,8 +53,8 @@ class CompletionOutput:
53
54
55 @dataclass
56 -class EmbeddingOutput:
57 - """The output data of one completion output of a request.
56 +class PoolingOutput:
57 + """The output data of one pooling output of a request.
58
59 Args:
60 embedding: The embedding vector, which is a list of floats. The
@@ -63,7 +63,7 @@ class EmbeddingOutput:
63 embedding: List[float]
64
65 def __repr__(self) -> str:
66 - return (f"EmbeddingOutput("
66 + return (f"PoolingOutput("
67 f"embedding={len(self.embedding)})")
68
69
@@ -316,18 +316,18 @@ class RequestOutput:
316 f"multi_modal_placeholders={self.multi_modal_placeholders})")
317
318
319 -class EmbeddingRequestOutput:
319 +class PoolingRequestOutput:
320 """
321 - The output data of an embedding request to the LLM.
321 + The output data of a pooling request to the LLM.
322
323 Args:
324 - request_id (str): A unique identifier for the embedding request.
325 - outputs (EmbeddingOutput): The embedding results for the given input.
324 + request_id (str): A unique identifier for the pooling request.
325 + outputs (PoolingOutput): The pooling results for the given input.
326 prompt_token_ids (List[int]): A list of token IDs used in the prompt.
327 - finished (bool): A flag indicating whether the embedding is completed.
327 + finished (bool): A flag indicating whether the pooling is completed.
328 """
329
330 - def __init__(self, request_id: str, outputs: "EmbeddingOutput",
330 + def __init__(self, request_id: str, outputs: "PoolingOutput",
331 prompt_token_ids: List[int], finished: bool):
332 self.request_id = request_id
333 self.prompt_token_ids = prompt_token_ids
@@ -336,11 +336,11 @@ class EmbeddingRequestOutput:
336
337 @classmethod
338 def from_seq_group(cls,
339 - seq_group: 'SequenceGroup') -> "EmbeddingRequestOutput":
339 + seq_group: 'SequenceGroup') -> "PoolingRequestOutput":
340 if seq_group.embeddings is None:
341 raise ValueError(
342 "Embeddings are missing in seq_group for EmbeddingRequest.")
343 - output = EmbeddingOutput(seq_group.embeddings)
343 + output = PoolingOutput(seq_group.embeddings)
344 prompt_token_ids = seq_group.prompt_token_ids
345 finished = seq_group.is_finished()
346
@@ -348,15 +348,15 @@ class EmbeddingRequestOutput:
348
349 def __repr__(self):
350 """
351 - Returns a string representation of an EmbeddingRequestOutput instance.
351 + Returns a string representation of an PoolingRequestOutput instance.
352
353 The representation includes the request_id and the number of outputs,
354 - providing a quick overview of the embedding request's results.
354 + providing a quick overview of the pooling request's results.
355
356 Returns:
357 - str: A string representation of the EmbeddingRequestOutput instance.
357 + str: A string representation of the PoolingRequestOutput instance.
358 """
359 - return (f"EmbeddingRequestOutput(request_id='{self.request_id}', "
359 + return (f"PoolingRequestOutput(request_id='{self.request_id}', "
360 f"outputs={repr(self.outputs)}, "
361 f"prompt_token_ids={self.prompt_token_ids}, "
362 f"finished={self.finished})")
@@ -415,7 +415,30 @@ class RequestOutputFactory:
415 # Determine the type based on a condition, for example:
416 if hasattr(seq_group,
417 'embeddings') and seq_group.embeddings is not None:
418 - return EmbeddingRequestOutput.from_seq_group(seq_group)
418 + return PoolingRequestOutput.from_seq_group(seq_group)
419 else:
420 return RequestOutput.from_seq_group(seq_group, use_cache,
421 seq_id_to_seq_group)
422 +
423 +
424 +def __getattr__(name: str):
425 + import warnings
426 +
427 + if name == "EmbeddingOutput":
428 + msg = ("EmbeddingOutput has been renamed to PoolingOutput. "
429 + "The original name will be removed in an upcoming version.")
430 +
431 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
432 +
433 + return PoolingOutput
434 +
435 + if name == "EmbeddingRequestOutput":
436 + msg = ("EmbeddingRequestOutput has been renamed to "
437 + "PoolingRequestOutput. "
438 + "The original name will be removed in an upcoming version.")
439 +
440 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
441 +
442 + return PoolingRequestOutput
443 +
444 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/v1/engine/async_llm.py
+2 -2
@@ -9,7 +9,7 @@ from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
9 from vllm.inputs.preprocess import InputPreprocessor
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 -from vllm.outputs import EmbeddingRequestOutput, RequestOutput
12 +from vllm.outputs import PoolingRequestOutput, RequestOutput
13 from vllm.pooling_params import PoolingParams
14 from vllm.prompt_adapter.request import PromptAdapterRequest
15 from vllm.sampling_params import SamplingParams
@@ -133,7 +133,7 @@ class AsyncLLM(EngineClient):
133 trace_headers: Optional[Mapping[str, str]] = None,
134 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
135 priority: int = 0,
136 - ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
136 + ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
137 """Add new request to the AsyncLLM."""
138
139 if self.detokenizer.is_request_active(request_id):
vllm/v1/engine/async_stream.py
+4 -4
@@ -1,11 +1,11 @@
1 import asyncio
2 from typing import Any, AsyncGenerator, Callable, Optional, Type, Union
3
4 -from vllm.outputs import EmbeddingRequestOutput, RequestOutput
4 +from vllm.outputs import PoolingRequestOutput, RequestOutput
5
6
7 class AsyncStream:
8 - """A stream of RequestOutputs or EmbeddingRequestOutputs for a request
8 + """A stream of RequestOutputs or PoolingRequestOutputs for a request
9 that can be iterated over asynchronously via an async generator."""
10
11 STOP_ITERATION = Exception() # Sentinel
@@ -16,7 +16,7 @@ class AsyncStream:
16 self._queue: asyncio.Queue = asyncio.Queue()
17 self._finished = False
18
19 - def put(self, item: Union[RequestOutput, EmbeddingRequestOutput,
19 + def put(self, item: Union[RequestOutput, PoolingRequestOutput,
20 Exception]) -> None:
21 if not self._finished:
22 self._queue.put_nowait(item)
@@ -32,7 +32,7 @@ class AsyncStream:
32
33 async def generator(
34 self
35 - ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
35 + ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
36 finished = False
37 try:
38 while True:
vllm/worker/cpu_pooling_model_runner.py renamed
+2 -2
@@ -16,12 +16,12 @@ from vllm.worker.cpu_model_runner import (CPUModelRunnerBase, ModelInputForCPU,
16 @dataclasses.dataclass(frozen=True)
17 class ModelInputForCPUWithPoolingMetadata(ModelInputForCPU):
18 """
19 - Used by the CPUEmbeddingModelRunner.
19 + Used by the CPUPoolingModelRunner.
20 """
21 pooling_metadata: Optional["PoolingMetadata"] = None
22
23
24 -class CPUEmbeddingModelRunner(
24 +class CPUPoolingModelRunner(
25 CPUModelRunnerBase[ModelInputForCPUWithPoolingMetadata]):
26 _model_input_cls: Type[ModelInputForCPUWithPoolingMetadata] = (
27 ModelInputForCPUWithPoolingMetadata)
vllm/worker/cpu_worker.py
+2 -2
@@ -14,9 +14,9 @@ from vllm.logger import init_logger
14 from vllm.model_executor import set_random_seed
15 from vllm.sequence import ExecuteModelRequest
16 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
17 -from vllm.worker.cpu_embedding_model_runner import CPUEmbeddingModelRunner
17 from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner
18 from vllm.worker.cpu_model_runner import CPUModelRunner, CPUModelRunnerBase
19 +from vllm.worker.cpu_pooling_model_runner import CPUPoolingModelRunner
20 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
21 LoraNotSupportedWorkerBase, WorkerBase,
22 WorkerInput)
@@ -164,7 +164,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
164 else {"return_hidden_states": True}
165 ModelRunnerClass: Type[CPUModelRunnerBase] = CPUModelRunner
166 if self.model_config.task == "embedding":
167 - ModelRunnerClass = CPUEmbeddingModelRunner
167 + ModelRunnerClass = CPUPoolingModelRunner
168 elif self.model_config.is_encoder_decoder:
169 ModelRunnerClass = CPUEncoderDecoderModelRunner
170 self.model_runner: CPUModelRunnerBase = ModelRunnerClass(
vllm/worker/pooling_model_runner.py renamed
+3 -3
@@ -21,12 +21,12 @@ logger = init_logger(__name__)
21 @dataclasses.dataclass(frozen=True)
22 class ModelInputForGPUWithPoolingMetadata(ModelInputForGPU):
23 """
24 - Used by the EmbeddingModelRunner.
24 + Used by the PoolingModelRunner.
25 """
26 pooling_metadata: Optional["PoolingMetadata"] = None
27
28
29 -class EmbeddingModelRunner(
29 +class PoolingModelRunner(
30 GPUModelRunnerBase[ModelInputForGPUWithPoolingMetadata]):
31 _model_input_cls: Type[ModelInputForGPUWithPoolingMetadata] = (
32 ModelInputForGPUWithPoolingMetadata)
@@ -52,7 +52,7 @@ class EmbeddingModelRunner(
52 ) -> Optional[Union[List[PoolerOutput], IntermediateTensors]]:
53 if num_steps > 1:
54 raise ValueError(
55 - "EmbeddingModelRunner does not support multi-step execution.")
55 + "PoolingModelRunner does not support multi-step execution.")
56
57 if self.lora_config:
58 assert model_input.lora_requests is not None
vllm/worker/worker.py
+2 -2
@@ -22,9 +22,9 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
22 from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
23 SequenceGroupMetadata, SequenceGroupMetadataDelta)
24 from vllm.worker.cache_engine import CacheEngine
25 -from vllm.worker.embedding_model_runner import EmbeddingModelRunner
25 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
26 from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
27 +from vllm.worker.pooling_model_runner import PoolingModelRunner
28 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, WorkerBase,
29 WorkerInput)
30
@@ -75,7 +75,7 @@ class Worker(LocalOrDistributedWorkerBase):
75
76 ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
77 if model_config.task == "embedding":
78 - ModelRunnerClass = EmbeddingModelRunner
78 + ModelRunnerClass = PoolingModelRunner
79 elif self.model_config.is_encoder_decoder:
80 ModelRunnerClass = EncoderDecoderModelRunner
81 self.model_runner: GPUModelRunnerBase = ModelRunnerClass(