@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cad5c0a6ed
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Doc] Update docs to refer to pooling models (#11093)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 11, 2024 at 21:36 UTC cad5c0a6eda057eeece87a42fff49fef3e18a2ac
14 files changed +26 -21
docs/source/usage/faq.rst
+6 -1
@@ -11,7 +11,12 @@ A: Assuming that you're referring to using OpenAI compatible server to serve mul
11
12 Q: Which model to use for offline inference embedding?
13
14 -A: If you want to use an embedding model, try: https://huggingface.co/intfloat/e5-mistral-7b-instruct. Instead models, such as Llama-3-8b, Mistral-7B-Instruct-v0.3, are generation models rather than an embedding model
14 +A: You can try `e5-mistral-7b-instruct <https://huggingface.co/intfloat/e5-mistral-7b-instruct>`__ and `BAAI/bge-base-en-v1.5 <https://huggingface.co/BAAI/bge-base-en-v1.5>`__;
15 +more are listed :ref:`here <supported_models>`.
16 +
17 +By extracting hidden states, vLLM can automatically convert text generation models like `Llama-3-8B <https://huggingface.co/meta-llama/Meta-Llama-3-8B>`__,
18 +`Mistral-7B-Instruct-v0.3 <https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3>`__ into embedding models,
19 +but they are expected be inferior to models that are specifically trained on embedding tasks.
20
21 ----------------------------------------
22
vllm/attention/backends/placeholder_attn.py
+1 -1
@@ -14,7 +14,7 @@ if TYPE_CHECKING:
14 from vllm.worker.model_runner import (ModelInputForGPUBuilder,
15 ModelInputForGPUWithSamplingMetadata)
16
17 -# Placeholder attention backend for models like Mamba and embedding models that
17 +# Placeholder attention backend for models like Mamba and pooling models that
18 # lack attention.
19
20
vllm/config.py
+4 -4
@@ -152,7 +152,7 @@ class ModelConfig:
152 this argument will be used to configure the neuron config that
153 can not be gathered from the vllm arguments.
154 override_pooler_config: Initialize non default pooling config or
155 - override default pooling config for the embedding model.
155 + override default pooling config for the pooling model.
156 """
157
158 def __init__(
@@ -576,7 +576,7 @@ class ModelConfig:
576 self.use_async_output_proc = False
577 return
578
579 - # Async postprocessor is not necessary with embedding mode
579 + # Async postprocessor is not necessary for pooling models
580 # since there is no token generation
581 if self.runner_type == "pooling":
582 self.use_async_output_proc = False
@@ -1825,11 +1825,11 @@ class MultiModalConfig:
1825
1826 @dataclass
1827 class PoolerConfig:
1828 - """Controls the behavior of output pooling in embedding models."""
1828 + """Controls the behavior of output pooling in pooling models."""
1829
1830 pooling_type: Optional[str] = None
1831 """
1832 - The pooling method of the embedding model. This should be a key in
1832 + The pooling method of the pooling model. This should be a key in
1833 :class:`vllm.model_executor.layers.pooler.PoolingType`.
1834 """
1835
vllm/core/placeholder_block_space_manager.py
+1 -1
@@ -8,7 +8,7 @@ from vllm.utils import Device
8 class PlaceholderBlockSpaceManager(BlockSpaceManager):
9 """A version of BlockSpaceManager for use in environments
10 where block management is not required.
11 - For example: embedding models or attention-free models like Mamba.
11 + For example: pooling models or attention-free models like Mamba.
12
13 This class provides the same interface as BlockSpaceManager, but its
14 methods perform no actions or return simple values like True in specific
vllm/engine/arg_utils.py
+2 -2
@@ -893,7 +893,7 @@ class EngineArgs:
893 '--override-pooler-config',
894 type=PoolerConfig.from_json,
895 default=None,
896 - help="Override or set the pooling method in the embedding model. "
896 + help="Override or set the pooling method for pooling models. "
897 "e.g. {\"pooling_type\": \"mean\", \"normalize\": false}.'")
898
899 parser.add_argument('--compilation-config',
@@ -1085,7 +1085,7 @@ class EngineArgs:
1085 "setting --max-model-len to a smaller value.", max_model_len)
1086 elif (self.enable_chunked_prefill
1087 and model_config.runner_type == "pooling"):
1088 - msg = "Chunked prefill is not supported for embedding models"
1088 + msg = "Chunked prefill is not supported for pooling models"
1089 raise ValueError(msg)
1090
1091
vllm/engine/async_llm_engine.py
+1 -1
@@ -1085,7 +1085,7 @@ class AsyncLLMEngine(EngineClient):
1085 trace_headers: Optional[Mapping[str, str]] = None,
1086 priority: int = 0,
1087 ) -> AsyncGenerator[PoolingRequestOutput, None]:
1088 - """Generate outputs for a request from an embedding model.
1088 + """Generate outputs for a request from a pooling model.
1089
1090 Generate outputs for a request. This method is a coroutine. It adds the
1091 request into the waiting queue of the LLMEngine and streams the outputs
vllm/engine/multiprocessing/client.py
+1 -1
@@ -527,7 +527,7 @@ class MQLLMEngineClient(EngineClient):
527 *,
528 inputs: Optional[PromptType] = None # DEPRECATED
529 ) -> AsyncGenerator[PoolingRequestOutput, None]:
530 - """Generate outputs for a request from an embedding model.
530 + """Generate outputs for a request from a pooling model.
531
532 Generate outputs for a request. This method is a coroutine. It adds the
533 request into the waiting queue of the LLMEngine and streams the outputs
vllm/engine/protocol.py
+1 -1
@@ -209,7 +209,7 @@ class EngineClient(ABC):
209 trace_headers: Optional[Mapping[str, str]] = None,
210 priority: int = 0,
211 ) -> AsyncGenerator[PoolingRequestOutput, None]:
212 - """Generate outputs for a request from an embedding model."""
212 + """Generate outputs for a request from a pooling model."""
213 ...
214
215 @abstractmethod
vllm/entrypoints/openai/serving_score.py
+1 -1
@@ -119,7 +119,7 @@ class OpenAIServingScores(OpenAIServing):
119
120 if prompt_adapter_request is not None:
121 raise NotImplementedError("Prompt adapter is not supported "
122 - "for embedding models")
122 + "for scoring models")
123
124 if isinstance(tokenizer, MistralTokenizer):
125 raise ValueError(
vllm/sequence.py
+3 -3
@@ -618,9 +618,9 @@ class SequenceGroup:
618 arrival_time: The arrival time of the request.
619 lora_request: LoRA request.
620 embeddings: The embeddings vectors of the prompt of the sequence group
621 - for an embedding model.
621 + for a pooling model.
622 pooling_params: The pooling parameters used to generate the pooling
623 - for an embedding model.
623 + for a pooling model.
624 encoder_seq: Optional, the single encoder sequence. Should be None
625 unless you are working with an encoder/decoder model.
626 trace_headers: OpenTelemetry trace headers.
@@ -1102,7 +1102,7 @@ class PoolerOutput(
1102 msgspec.Struct,
1103 omit_defaults=True, # type: ignore[call-arg]
1104 array_like=True): # type: ignore[call-arg]
1105 - """The output from a pooling operation in the embedding model."""
1105 + """The output from a pooling operation in the pooling model."""
1106 outputs: List[EmbeddingSequenceGroupOutput]
1107
1108 # lazy import to avoid circular import
vllm/v1/engine/processor.py
+1 -1
@@ -59,7 +59,7 @@ class Processor:
59 priority: int = 0,
60 ) -> Tuple[DetokenizerRequest, EngineCoreRequest]:
61
62 - # TODO(woosuk): Support embedding mode.
62 + # TODO(woosuk): Support pooling models.
63 # TODO(woosuk): Check max_logprobs
64 # TODO(woosuk): Support encoder-decoder models.
65
vllm/worker/cpu_worker.py
+1 -1
@@ -178,7 +178,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
178 # Uninitialized cache engine. Will be initialized by
179 # initialize_cache.
180 self.cache_engine: List[CPUCacheEngine]
181 - # Initialize cpu_cache as embedding models don't initialize kv_caches
181 + # Initialize cpu_cache as pooling models don't initialize kv_caches
182 self.cpu_cache: Optional[List[List[torch.Tensor]]] = None
183
184 # Torch profiler. Enabled and configured through env vars:
vllm/worker/hpu_worker.py
+2 -2
@@ -65,8 +65,8 @@ class HPUWorker(LocalOrDistributedWorkerBase):
65 # Uninitialized cache engine. Will be initialized by
66 # initialize_cache.
67 self.cache_engine: List[HPUCacheEngine]
68 - # Initialize gpu_cache as embedding models don't initialize kv_caches
69 - self.hpu_cache: Optional[List[List[torch.tensor]]] = None
68 + # Initialize gpu_cache as pooling models don't initialize kv_caches
69 + self.hpu_cache: Optional[List[List[torch.Tensor]]] = None
70 # Torch profiler. Enabled and configured through env vars:
71 # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
72 if envs.VLLM_TORCH_PROFILER_DIR:
vllm/worker/worker.py
+1 -1
@@ -91,7 +91,7 @@ class Worker(LocalOrDistributedWorkerBase):
91 # Uninitialized cache engine. Will be initialized by
92 # initialize_cache.
93 self.cache_engine: List[CacheEngine]
94 - # Initialize gpu_cache as embedding models don't initialize kv_caches
94 + # Initialize gpu_cache as pooling models don't initialize kv_caches
95 self.gpu_cache: Optional[List[List[torch.Tensor]]] = None
96 self._seq_group_metadata_cache: Dict[str, SequenceGroupMetadata] = {}
97