A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Doc] Update docs to refer to pooling models (#11093)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 11, 2024 at 21:36 UTC
cad5c0a6eda057eeece87a42fff49fef3e18a2ac
14 files changed
+26
-21
docs/source/usage/faq.rst
+6
-1
@@ -11,7 +11,12 @@ A: Assuming that you're referring to using OpenAI compatible server to serve mul
11
12
Q: Which model to use for offline inference embedding?
13
14
-A: If you want to use an embedding model, try: https://huggingface.co/intfloat/e5-mistral-7b-instruct. Instead models, such as Llama-3-8b, Mistral-7B-Instruct-v0.3, are generation models rather than an embedding model
14
+A: You can try `e5-mistral-7b-instruct <https://huggingface.co/intfloat/e5-mistral-7b-instruct>`__ and `BAAI/bge-base-en-v1.5 <https://huggingface.co/BAAI/bge-base-en-v1.5>`__;
15
+more are listed :ref:`here <supported_models>`.
16
+
17
+By extracting hidden states, vLLM can automatically convert text generation models like `Llama-3-8B <https://huggingface.co/meta-llama/Meta-Llama-3-8B>`__,
18
+`Mistral-7B-Instruct-v0.3 <https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3>`__ into embedding models,
19
+but they are expected be inferior to models that are specifically trained on embedding tasks.
20
21
----------------------------------------
22
vllm/attention/backends/placeholder_attn.py
+1
-1
@@ -14,7 +14,7 @@ if TYPE_CHECKING:
14
from vllm.worker.model_runner import (ModelInputForGPUBuilder,
15
ModelInputForGPUWithSamplingMetadata)
16
17
-# Placeholder attention backend for models like Mamba and embedding models that
17
+# Placeholder attention backend for models like Mamba and pooling models that
18
# lack attention.
19
20
vllm/config.py
+4
-4
@@ -152,7 +152,7 @@ class ModelConfig:
152
this argument will be used to configure the neuron config that
153
can not be gathered from the vllm arguments.
154
override_pooler_config: Initialize non default pooling config or
155
- override default pooling config for the embedding model.
155
+ override default pooling config for the pooling model.
156
"""
157
158
def __init__(
@@ -576,7 +576,7 @@ class ModelConfig:
576
self.use_async_output_proc = False
577
return
578
579
- # Async postprocessor is not necessary with embedding mode
579
+ # Async postprocessor is not necessary for pooling models
580
# since there is no token generation
581
if self.runner_type == "pooling":
582
self.use_async_output_proc = False
@@ -1825,11 +1825,11 @@ class MultiModalConfig:
1825
1826
@dataclass
1827
class PoolerConfig:
1828
- """Controls the behavior of output pooling in embedding models."""
1828
+ """Controls the behavior of output pooling in pooling models."""
1829
1830
pooling_type: Optional[str] = None
1831
"""
1832
- The pooling method of the embedding model. This should be a key in
1832
+ The pooling method of the pooling model. This should be a key in
1833
:class:`vllm.model_executor.layers.pooler.PoolingType`.
1834
"""
1835
vllm/core/placeholder_block_space_manager.py
+1
-1
@@ -8,7 +8,7 @@ from vllm.utils import Device
8
class PlaceholderBlockSpaceManager(BlockSpaceManager):
9
"""A version of BlockSpaceManager for use in environments
10
where block management is not required.
11
- For example: embedding models or attention-free models like Mamba.
11
+ For example: pooling models or attention-free models like Mamba.
12
13
This class provides the same interface as BlockSpaceManager, but its
14
methods perform no actions or return simple values like True in specific
vllm/engine/arg_utils.py
+2
-2
@@ -893,7 +893,7 @@ class EngineArgs:
893
'--override-pooler-config',
894
type=PoolerConfig.from_json,
895
default=None,
896
- help="Override or set the pooling method in the embedding model. "
896
+ help="Override or set the pooling method for pooling models. "
897
"e.g. {\"pooling_type\": \"mean\", \"normalize\": false}.'")
898
899
parser.add_argument('--compilation-config',
@@ -1085,7 +1085,7 @@ class EngineArgs:
1085
"setting --max-model-len to a smaller value.", max_model_len)
1086
elif (self.enable_chunked_prefill
1087
and model_config.runner_type == "pooling"):
1088
- msg = "Chunked prefill is not supported for embedding models"
1088
+ msg = "Chunked prefill is not supported for pooling models"
1089
raise ValueError(msg)
1090
1091
vllm/engine/async_llm_engine.py
+1
-1
@@ -1085,7 +1085,7 @@ class AsyncLLMEngine(EngineClient):
1085
trace_headers: Optional[Mapping[str, str]] = None,
1086
priority: int = 0,
1087
) -> AsyncGenerator[PoolingRequestOutput, None]:
1088
- """Generate outputs for a request from an embedding model.
1088
+ """Generate outputs for a request from a pooling model.
1089
1090
Generate outputs for a request. This method is a coroutine. It adds the
1091
request into the waiting queue of the LLMEngine and streams the outputs
vllm/engine/multiprocessing/client.py
+1
-1
@@ -527,7 +527,7 @@ class MQLLMEngineClient(EngineClient):
527
*,
528
inputs: Optional[PromptType] = None # DEPRECATED
529
) -> AsyncGenerator[PoolingRequestOutput, None]:
530
- """Generate outputs for a request from an embedding model.
530
+ """Generate outputs for a request from a pooling model.
531
532
Generate outputs for a request. This method is a coroutine. It adds the
533
request into the waiting queue of the LLMEngine and streams the outputs
vllm/engine/protocol.py
+1
-1
@@ -209,7 +209,7 @@ class EngineClient(ABC):
209
trace_headers: Optional[Mapping[str, str]] = None,
210
priority: int = 0,
211
) -> AsyncGenerator[PoolingRequestOutput, None]:
212
- """Generate outputs for a request from an embedding model."""
212
+ """Generate outputs for a request from a pooling model."""
213
...
214
215
@abstractmethod
vllm/entrypoints/openai/serving_score.py
+1
-1
@@ -119,7 +119,7 @@ class OpenAIServingScores(OpenAIServing):
119
120
if prompt_adapter_request is not None:
121
raise NotImplementedError("Prompt adapter is not supported "
122
- "for embedding models")
122
+ "for scoring models")
123
124
if isinstance(tokenizer, MistralTokenizer):
125
raise ValueError(
vllm/sequence.py
+3
-3
@@ -618,9 +618,9 @@ class SequenceGroup:
618
arrival_time: The arrival time of the request.
619
lora_request: LoRA request.
620
embeddings: The embeddings vectors of the prompt of the sequence group
621
- for an embedding model.
621
+ for a pooling model.
622
pooling_params: The pooling parameters used to generate the pooling
623
- for an embedding model.
623
+ for a pooling model.
624
encoder_seq: Optional, the single encoder sequence. Should be None
625
unless you are working with an encoder/decoder model.
626
trace_headers: OpenTelemetry trace headers.
@@ -1102,7 +1102,7 @@ class PoolerOutput(
1102
msgspec.Struct,
1103
omit_defaults=True, # type: ignore[call-arg]
1104
array_like=True): # type: ignore[call-arg]
1105
- """The output from a pooling operation in the embedding model."""
1105
+ """The output from a pooling operation in the pooling model."""
1106
outputs: List[EmbeddingSequenceGroupOutput]
1107
1108
# lazy import to avoid circular import
vllm/v1/engine/processor.py
+1
-1
@@ -59,7 +59,7 @@ class Processor:
59
priority: int = 0,
60
) -> Tuple[DetokenizerRequest, EngineCoreRequest]:
61
62
- # TODO(woosuk): Support embedding mode.
62
+ # TODO(woosuk): Support pooling models.
63
# TODO(woosuk): Check max_logprobs
64
# TODO(woosuk): Support encoder-decoder models.
65
vllm/worker/cpu_worker.py
+1
-1
@@ -178,7 +178,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
178
# Uninitialized cache engine. Will be initialized by
179
# initialize_cache.
180
self.cache_engine: List[CPUCacheEngine]
181
- # Initialize cpu_cache as embedding models don't initialize kv_caches
181
+ # Initialize cpu_cache as pooling models don't initialize kv_caches
182
self.cpu_cache: Optional[List[List[torch.Tensor]]] = None
183
184
# Torch profiler. Enabled and configured through env vars:
vllm/worker/hpu_worker.py
+2
-2
@@ -65,8 +65,8 @@ class HPUWorker(LocalOrDistributedWorkerBase):
65
# Uninitialized cache engine. Will be initialized by
66
# initialize_cache.
67
self.cache_engine: List[HPUCacheEngine]
68
- # Initialize gpu_cache as embedding models don't initialize kv_caches
69
- self.hpu_cache: Optional[List[List[torch.tensor]]] = None
68
+ # Initialize gpu_cache as pooling models don't initialize kv_caches
69
+ self.hpu_cache: Optional[List[List[torch.Tensor]]] = None
70
# Torch profiler. Enabled and configured through env vars:
71
# VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
72
if envs.VLLM_TORCH_PROFILER_DIR:
vllm/worker/worker.py
+1
-1
@@ -91,7 +91,7 @@ class Worker(LocalOrDistributedWorkerBase):
91
# Uninitialized cache engine. Will be initialized by
92
# initialize_cache.
93
self.cache_engine: List[CacheEngine]
94
- # Initialize gpu_cache as embedding models don't initialize kv_caches
94
+ # Initialize gpu_cache as pooling models don't initialize kv_caches
95
self.gpu_cache: Optional[List[List[torch.Tensor]]] = None
96
self._seq_group_metadata_cache: Dict[str, SequenceGroupMetadata] = {}
97