@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 058525b997
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Move `PoolerConfig` from `config/__init__.py` to `config/pooler.py` (#25181)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Sep 19, 2025 at 12:02 UTC 058525b9973cabfe27b7ab34dad6dbcbb6859f74
14 files changed +193 -179
docs/models/pooling_models.md
+2 -2
@@ -59,7 +59,7 @@ enabling the corresponding APIs:
59 #### Predefined models
60
61 If the [Pooler][vllm.model_executor.layers.pooler.Pooler] defined by the model accepts `pooler_config`,
62 -you can override some of its attributes via the `--override-pooler-config` option.
62 +you can override some of its attributes via the `--pooler-config` option.
63
64 #### Converted models
65
@@ -75,7 +75,7 @@ the pooler assigned to each task has the following attributes by default:
75 When loading [Sentence Transformers](https://huggingface.co/sentence-transformers) models,
76 its Sentence Transformers configuration file (`modules.json`) takes priority over the model's defaults.
77
78 -You can further customize this via the `--override-pooler-config` option,
78 +You can further customize this via the `--pooler-config` option,
79 which takes priority over both the model's and Sentence Transformers's defaults.
80
81 ## Offline Inference
docs/models/supported_models.md
+2 -2
@@ -457,7 +457,7 @@ These models primarily support the [`LLM.embed`](./pooling_models.md#llmembed) A
457
458 !!! note
459 `ssmits/Qwen2-7B-Instruct-embed-base` has an improperly defined Sentence Transformers config.
460 - You need to manually set mean pooling by passing `--override-pooler-config '{"pooling_type": "MEAN"}'`.
460 + You need to manually set mean pooling by passing `--pooler-config '{"pooling_type": "MEAN"}'`.
461
462 !!! note
463 For `Alibaba-NLP/gte-Qwen2-*`, you need to enable `--trust-remote-code` for the correct tokenizer to be loaded.
@@ -552,7 +552,7 @@ If your model is not in the above list, we will try to automatically convert the
552
553 !!! important
554 For process-supervised reward models such as `peiyi9979/math-shepherd-mistral-7b-prm`, the pooling config should be set explicitly,
555 - e.g.: `--override-pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
555 + e.g.: `--pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
556
557 #### Token Classification
558
examples/online_serving/openai_embedding_long_text/README.md
+1 -1
@@ -42,7 +42,7 @@ python client.py
42
43 ### Server Configuration
44
45 -The key parameters for chunked processing are in the `--override-pooler-config`:
45 +The key parameters for chunked processing are in the `--pooler-config`:
46
47 ```json
48 {
examples/online_serving/openai_embedding_long_text/client.py
+2 -2
@@ -13,7 +13,7 @@ Prerequisites:
13
14 # MEAN pooling (processes all chunks, recommended for complete coverage)
15 vllm serve intfloat/multilingual-e5-large \
16 - --override-pooler-config \
16 + --pooler-config \
17 '{"pooling_type": "MEAN", "normalize": true, ' \
18 '"enable_chunked_processing": true, "max_embed_len": 3072000}' \
19 --served-model-name multilingual-e5-large \
@@ -23,7 +23,7 @@ Prerequisites:
23
24 # OR CLS pooling (native CLS within chunks, MEAN aggregation across chunks)
25 vllm serve BAAI/bge-large-en-v1.5 \
26 - --override-pooler-config \
26 + --pooler-config \
27 '{"pooling_type": "CLS", "normalize": true, ' \
28 '"enable_chunked_processing": true, "max_embed_len": 1048576}' \
29 --served-model-name bge-large-en-v1.5 \
examples/online_serving/openai_embedding_long_text/service.sh
+1 -1
@@ -103,7 +103,7 @@ POOLER_CONFIG="{\"pooling_type\": \"$POOLING_TYPE\", \"normalize\": true, \"enab
103 vllm serve "$MODEL_NAME" \
104 --tensor-parallel-size "$GPU_COUNT" \
105 --enforce-eager \
106 - --override-pooler-config "$POOLER_CONFIG" \
106 + --pooler-config "$POOLER_CONFIG" \
107 --served-model-name ${MODEL_CODE} \
108 --api-key "$API_KEY" \
109 --trust-remote-code \
tests/entrypoints/pooling/openai/test_embedding_long_text.py
+1 -1
@@ -216,7 +216,7 @@ def server_with_chunked_processing():
216 "--enforce-eager",
217 "--max-model-len",
218 "512", # Set smaller max_model_len to trigger chunking mechanism
219 - '--override-pooler-config',
219 + '--pooler-config',
220 ('{"pooling_type": "MEAN", "normalize": true, '
221 '"enable_chunked_processing": true, "max_embed_len": 10000}'),
222 "--gpu-memory-utilization",
tests/models/language/pooling/test_embedding.py
+1 -1
@@ -58,7 +58,7 @@ def test_models(
58
59 vllm_extra_kwargs = {}
60 if model == "ssmits/Qwen2-7B-Instruct-embed-base":
61 - vllm_extra_kwargs["override_pooler_config"] = \
61 + vllm_extra_kwargs["pooler_config"] = \
62 PoolerConfig(pooling_type="MEAN", normalize=False)
63
64 max_model_len: Optional[int] = 512
tests/models/language/pooling/test_mm_classifier_conversion.py
+2 -1
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +from vllm.config.pooler import PoolerConfig
5 from vllm.platforms import current_platform
6
7
@@ -99,7 +100,7 @@ def test_gemma_multimodal(
100 convert="classify",
101 load_format="auto",
102 hf_overrides=update_config,
102 - override_pooler_config={"pooling_type": "LAST"},
103 + pooler_config=PoolerConfig(pooling_type="LAST"),
104 max_model_len=512,
105 enforce_eager=True,
106 tensor_parallel_size=1,
tests/models/language/pooling/test_pooler_config_init_behaviour.py renamed
+29 -33
@@ -24,18 +24,18 @@ def test_classify_models_using_activation(
24 dtype: str,
25 ) -> None:
26
27 - with vllm_runner(model,
28 - max_model_len=512,
29 - dtype=dtype,
30 - override_pooler_config=PoolerConfig(
31 - activation=False)) as vllm_model:
27 + with vllm_runner(
28 + model,
29 + max_model_len=512,
30 + dtype=dtype,
31 + pooler_config=PoolerConfig(activation=False)) as vllm_model:
32 wo_activation_out = vllm_model.classify(example_prompts)
33
34 - with vllm_runner(model,
35 - max_model_len=512,
36 - dtype=dtype,
37 - override_pooler_config=PoolerConfig(
38 - activation=True)) as vllm_model:
34 + with vllm_runner(
35 + model,
36 + max_model_len=512,
37 + dtype=dtype,
38 + pooler_config=PoolerConfig(activation=True)) as vllm_model:
39 w_activation_out = vllm_model.classify(example_prompts)
40
41 for wo_activation, w_activation in zip(wo_activation_out,
@@ -43,9 +43,8 @@ def test_classify_models_using_activation(
43 wo_activation = torch.tensor(wo_activation)
44 w_activation = torch.tensor(w_activation)
45
46 - assert not torch.allclose(
47 - wo_activation, w_activation,
48 - atol=1e-2), "override_pooler_config is not working"
46 + assert not torch.allclose(wo_activation, w_activation,
47 + atol=1e-2), "pooler_config is not working"
48 assert torch.allclose(softmax(wo_activation), w_activation,
49 1e-3 if dtype == "float" else 1e-2)
50
@@ -65,23 +64,22 @@ def test_embed_models_using_normalize(
64 dtype: str,
65 ) -> None:
66
68 - with vllm_runner(model,
69 - max_model_len=512,
70 - dtype=dtype,
71 - override_pooler_config=PoolerConfig(
72 - normalize=False)) as vllm_model:
73 - wo_normalize = torch.tensor(vllm_model.embed(example_prompts))
74 -
67 with vllm_runner(
68 model,
69 max_model_len=512,
70 dtype=dtype,
79 - override_pooler_config=PoolerConfig(normalize=True)) as vllm_model:
71 + pooler_config=PoolerConfig(normalize=False)) as vllm_model:
72 + wo_normalize = torch.tensor(vllm_model.embed(example_prompts))
73 +
74 + with vllm_runner(model,
75 + max_model_len=512,
76 + dtype=dtype,
77 + pooler_config=PoolerConfig(normalize=True)) as vllm_model:
78 w_normalize = torch.tensor(vllm_model.embed(example_prompts))
79
80 assert not torch.allclose(
81 wo_normalize, w_normalize,
84 - atol=1e-2), "override_pooler_config normalize is not working"
82 + atol=1e-2), "pooler_config normalize is not working"
83 assert torch.allclose(
84 F.normalize(wo_normalize, p=2, dim=-1), w_normalize,
85 atol=1e-2), "w_normal should be close to normal(wo_normal)."
@@ -102,18 +100,16 @@ def test_reward_models_using_softmax(
100 dtype: str,
101 ) -> None:
102
105 - with vllm_runner(
106 - model,
107 - max_model_len=1024,
108 - dtype=dtype,
109 - override_pooler_config=PoolerConfig(softmax=False)) as vllm_model:
103 + with vllm_runner(model,
104 + max_model_len=1024,
105 + dtype=dtype,
106 + pooler_config=PoolerConfig(softmax=False)) as vllm_model:
107 wo_softmax = vllm_model.encode(example_prompts)
108
112 - with vllm_runner(
113 - model,
114 - max_model_len=1024,
115 - dtype=dtype,
116 - override_pooler_config=PoolerConfig(softmax=True)) as vllm_model:
109 + with vllm_runner(model,
110 + max_model_len=1024,
111 + dtype=dtype,
112 + pooler_config=PoolerConfig(softmax=True)) as vllm_model:
113 w_softmax = vllm_model.encode(example_prompts)
114
115 for wo, w in zip(wo_softmax, w_softmax):
@@ -121,7 +117,7 @@ def test_reward_models_using_softmax(
117 w = torch.tensor(w)
118
119 assert not torch.allclose(
124 - wo, w, atol=1e-2), "override_pooler_config softmax is not working"
120 + wo, w, atol=1e-2), "pooler_config softmax is not working"
121 assert torch.allclose(
122 softmax(wo), w,
123 atol=1e-2), "w_softmax should be close to softmax(wo_softmax)."
tests/test_config.py
+6 -12
@@ -207,25 +207,19 @@ def test_get_pooling_config():
207 model_id = "sentence-transformers/all-MiniLM-L12-v2"
208 model_config = ModelConfig(model_id)
209
210 - pooling_config = model_config._init_pooler_config()
211 - assert pooling_config is not None
212 -
213 - assert pooling_config.normalize
214 - assert pooling_config.pooling_type == PoolingType.MEAN.name
210 + assert model_config.pooler_config is not None
211 + assert model_config.pooler_config.normalize
212 + assert model_config.pooler_config.pooling_type == PoolingType.MEAN.name
213
214
215 @pytest.mark.skipif(current_platform.is_rocm(),
216 reason="Xformers backend is not supported on ROCm.")
217 def test_get_pooling_config_from_args():
218 model_id = "sentence-transformers/all-MiniLM-L12-v2"
221 - model_config = ModelConfig(model_id)
222 -
223 - override_pooler_config = PoolerConfig(pooling_type='CLS', normalize=True)
224 - model_config.override_pooler_config = override_pooler_config
219 + pooler_config = PoolerConfig(pooling_type="CLS", normalize=True)
220 + model_config = ModelConfig(model_id, pooler_config=pooler_config)
221
226 - pooling_config = model_config._init_pooler_config()
227 - assert pooling_config is not None
228 - assert asdict(pooling_config) == asdict(override_pooler_config)
222 + assert asdict(model_config.pooler_config) == asdict(pooler_config)
223
224
225 @pytest.mark.parametrize(
vllm/config/__init__.py
+36 -119
@@ -40,6 +40,7 @@ from vllm.config.multimodal import (MMCacheType, MMEncoderTPMode,
40 MultiModalConfig)
41 from vllm.config.parallel import (DistributedExecutorBackend, EPLBConfig,
42 ParallelConfig)
43 +from vllm.config.pooler import PoolerConfig
44 from vllm.config.scheduler import SchedulerConfig, SchedulerPolicy
45 from vllm.config.speculative import SpeculativeConfig
46 from vllm.config.structured_outputs import StructuredOutputsConfig
@@ -406,13 +407,6 @@ class ModelConfig:
407 hf_overrides: HfOverrides = field(default_factory=dict)
408 """If a dictionary, contains arguments to be forwarded to the Hugging Face
409 config. If a callable, it is called to update the HuggingFace config."""
409 - pooler_config: Optional["PoolerConfig"] = field(init=False)
410 - """Pooler config which controls the behaviour of output pooling in pooling
411 - models."""
412 - override_pooler_config: Optional[Union[dict, "PoolerConfig"]] = None
413 - """Initialize non-default pooling config or override default pooling config
414 - for the pooling model. e.g. `{"pooling_type": "mean", "normalize": false}`.
415 - """
410 logits_processor_pattern: Optional[str] = None
411 """Optional regex pattern specifying valid logits processor qualified names
412 that can be passed with the `logits_processors` extra completion argument.
@@ -448,6 +442,14 @@ class ModelConfig:
442 io_processor_plugin: Optional[str] = None
443 """IOProcessor plugin name to load at model startup"""
444
445 + # Pooler config
446 + pooler_config: Optional[PoolerConfig] = None
447 + """Pooler config which controls the behaviour of output pooling in pooling
448 + models."""
449 + override_pooler_config: Optional[Union[dict, PoolerConfig]] = None
450 + """[DEPRECATED] Use `pooler_config` instead. This field will be removed in
451 + v0.12.0 or v1.0.0, whichever is sooner."""
452 +
453 # Multimodal config and init vars
454 multimodal_config: Optional[MultiModalConfig] = None
455 """Configuration for multimodal model. If `None`, this will be inferred
@@ -709,7 +711,33 @@ class ModelConfig:
711 self._architecture = arch
712 logger.info("Resolved architecture: %s", arch)
713
712 - self.pooler_config = self._init_pooler_config()
714 + # Init pooler config if needed
715 + if self.runner_type == "pooling":
716 + if self.override_pooler_config is not None:
717 + logger.warning_once(
718 + "`override_pooler_config` is deprecated and will be "
719 + "removed in v0.12.0 or v1.0.0, whichever is sooner. "
720 + "Please use `pooler_config` instead.")
721 +
722 + if isinstance(self.override_pooler_config, dict):
723 + self.pooler_config = PoolerConfig(
724 + **self.override_pooler_config)
725 + else:
726 + self.pooler_config = self.override_pooler_config
727 +
728 + if self.pooler_config is None:
729 + self.pooler_config = PoolerConfig()
730 +
731 + base_config = get_pooling_config(self.model, self.revision)
732 + if base_config is not None:
733 + # Only set values that are not overridden by the user
734 + for k, v in base_config.items():
735 + if getattr(self.pooler_config, k) is None:
736 + setattr(self.pooler_config, k, v)
737 +
738 + default_pooling_type = self._model_info.default_pooling_type
739 + if self.pooler_config.pooling_type is None:
740 + self.pooler_config.pooling_type = default_pooling_type
741
742 self.dtype: torch.dtype = _get_and_verify_dtype(
743 self.model,
@@ -869,29 +897,6 @@ class ModelConfig:
897 return get_sentence_transformer_tokenizer_config(
898 self.model, self.revision)
899
872 - def _init_pooler_config(self) -> Optional["PoolerConfig"]:
873 - if self.runner_type == "pooling":
874 - if isinstance(self.override_pooler_config, dict):
875 - self.override_pooler_config = PoolerConfig(
876 - **self.override_pooler_config)
877 -
878 - pooler_config = self.override_pooler_config or PoolerConfig()
879 -
880 - base_config = get_pooling_config(self.model, self.revision)
881 - if base_config is not None:
882 - # Only set values that are not overridden by the user
883 - for k, v in base_config.items():
884 - if getattr(pooler_config, k) is None:
885 - setattr(pooler_config, k, v)
886 -
887 - default_pooling_type = self._model_info.default_pooling_type
888 - if pooler_config.pooling_type is None:
889 - pooler_config.pooling_type = default_pooling_type
890 -
891 - return pooler_config
892 -
893 - return None
894 -
900 def _verify_tokenizer_mode(self) -> None:
901 tokenizer_mode = cast(TokenizerMode, self.tokenizer_mode.lower())
902 if tokenizer_mode not in get_args(TokenizerMode):
@@ -1833,94 +1838,6 @@ class DeviceConfig:
1838 self.device = torch.device(self.device_type)
1839
1840
1836 -@config
1837 -@dataclass
1838 -class PoolerConfig:
1839 - """Controls the behavior of output pooling in pooling models."""
1840 -
1841 - pooling_type: Optional[str] = None
1842 - """
1843 - The pooling method of the pooling model. This should be a key in
1844 - [`vllm.model_executor.layers.pooler.PoolingType`][].
1845 - """
1846 -
1847 - ## for embeddings models
1848 - normalize: Optional[bool] = None
1849 - """
1850 - Whether to normalize the embeddings outputs. Defaults to True.
1851 - """
1852 - dimensions: Optional[int] = None
1853 - """
1854 - Reduce the dimensions of embeddings if model
1855 - support matryoshka representation. Defaults to None.
1856 - """
1857 - enable_chunked_processing: Optional[bool] = None
1858 - """
1859 - Whether to enable chunked processing for long inputs that exceed the model's
1860 - maximum position embeddings. When enabled, long inputs will be split into
1861 - chunks, processed separately, and then aggregated using weighted averaging.
1862 - This allows embedding models to handle arbitrarily long text without CUDA
1863 - errors. Defaults to False.
1864 - """
1865 - max_embed_len: Optional[int] = None
1866 - """
1867 - Maximum input length allowed for embedding generation. When set, allows
1868 - inputs longer than max_embed_len to be accepted for embedding models.
1869 - When an input exceeds max_embed_len, it will be handled according to
1870 - the original max_model_len validation logic.
1871 - Defaults to None (i.e. set to max_model_len).
1872 - """
1873 -
1874 - ## for classification models
1875 - activation: Optional[bool] = None
1876 - """
1877 - Whether to apply activation function to the classification outputs.
1878 - Defaults to True.
1879 - """
1880 - logit_bias: Optional[float] = None
1881 - """
1882 - If provided, apply classification logit biases. Defaults to None.
1883 - """
1884 -
1885 - ## for reward models
1886 - softmax: Optional[bool] = None
1887 - """
1888 - Whether to apply softmax to the reward outputs.
1889 - Defaults to True.
1890 - """
1891 - step_tag_id: Optional[int] = None
1892 - """
1893 - If set, only the score corresponding to the ``step_tag_id`` in the
1894 - generated sentence should be returned. Otherwise, the scores for all tokens
1895 - are returned.
1896 - """
1897 - returned_token_ids: Optional[list[int]] = None
1898 - """
1899 - A list of indices for the vocabulary dimensions to be extracted,
1900 - such as the token IDs of ``good_token`` and ``bad_token`` in the
1901 - ``math-shepherd-mistral-7b-prm`` model.
1902 - """
1903 -
1904 - def compute_hash(self) -> str:
1905 - """
1906 - WARNING: Whenever a new field is added to this config,
1907 - ensure that it is included in the factors list if
1908 - it affects the computation graph.
1909 -
1910 - Provide a hash that uniquely identifies all the configs
1911 - that affect the structure of the computation
1912 - graph from input ids/embeddings to the final hidden states,
1913 - excluding anything before input ids/embeddings and after
1914 - the final hidden states.
1915 - """
1916 - # no factors to consider.
1917 - # this config will not affect the computation graph.
1918 - factors: list[Any] = []
1919 - hash_str = hashlib.md5(str(factors).encode(),
1920 - usedforsecurity=False).hexdigest()
1921 - return hash_str
1922 -
1923 -
1841 _STR_DTYPE_TO_TORCH_DTYPE = {
1842 "half": torch.float16,
1843 "float16": torch.float16,
vllm/config/pooler.py new
+97
@@ -0,0 +1,97 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import hashlib
5 +from typing import Any, Optional
6 +
7 +from pydantic.dataclasses import dataclass
8 +
9 +from vllm.config.utils import config
10 +
11 +
12 +@config
13 +@dataclass
14 +class PoolerConfig:
15 + """Controls the behavior of output pooling in pooling models."""
16 +
17 + pooling_type: Optional[str] = None
18 + """
19 + The pooling method of the pooling model. This should be a key in
20 + [`vllm.model_executor.layers.pooler.PoolingType`][].
21 + """
22 +
23 + ## for embeddings models
24 + normalize: Optional[bool] = None
25 + """
26 + Whether to normalize the embeddings outputs. Defaults to True.
27 + """
28 + dimensions: Optional[int] = None
29 + """
30 + Reduce the dimensions of embeddings if model
31 + support matryoshka representation. Defaults to None.
32 + """
33 + enable_chunked_processing: Optional[bool] = None
34 + """
35 + Whether to enable chunked processing for long inputs that exceed the model's
36 + maximum position embeddings. When enabled, long inputs will be split into
37 + chunks, processed separately, and then aggregated using weighted averaging.
38 + This allows embedding models to handle arbitrarily long text without CUDA
39 + errors. Defaults to False.
40 + """
41 + max_embed_len: Optional[int] = None
42 + """
43 + Maximum input length allowed for embedding generation. When set, allows
44 + inputs longer than max_embed_len to be accepted for embedding models.
45 + When an input exceeds max_embed_len, it will be handled according to
46 + the original max_model_len validation logic.
47 + Defaults to None (i.e. set to max_model_len).
48 + """
49 +
50 + ## for classification models
51 + activation: Optional[bool] = None
52 + """
53 + Whether to apply activation function to the classification outputs.
54 + Defaults to True.
55 + """
56 + logit_bias: Optional[float] = None
57 + """
58 + If provided, apply classification logit biases. Defaults to None.
59 + """
60 +
61 + ## for reward models
62 + softmax: Optional[bool] = None
63 + """
64 + Whether to apply softmax to the reward outputs.
65 + Defaults to True.
66 + """
67 + step_tag_id: Optional[int] = None
68 + """
69 + If set, only the score corresponding to the ``step_tag_id`` in the
70 + generated sentence should be returned. Otherwise, the scores for all tokens
71 + are returned.
72 + """
73 + returned_token_ids: Optional[list[int]] = None
74 + """
75 + A list of indices for the vocabulary dimensions to be extracted,
76 + such as the token IDs of ``good_token`` and ``bad_token`` in the
77 + ``math-shepherd-mistral-7b-prm`` model.
78 + """
79 +
80 + def compute_hash(self) -> str:
81 + """
82 + WARNING: Whenever a new field is added to this config,
83 + ensure that it is included in the factors list if
84 + it affects the computation graph.
85 +
86 + Provide a hash that uniquely identifies all the configs
87 + that affect the structure of the computation
88 + graph from input ids/embeddings to the final hidden states,
89 + excluding anything before input ids/embeddings and after
90 + the final hidden states.
91 + """
92 + # no factors to consider.
93 + # this config will not affect the computation graph.
94 + factors: list[Any] = []
95 + hash_str = hashlib.md5(str(factors).encode(),
96 + usedforsecurity=False).hexdigest()
97 + return hash_str
vllm/engine/arg_utils.py
+6 -1
@@ -441,6 +441,7 @@ class EngineArgs:
441 scheduling_policy: SchedulerPolicy = SchedulerConfig.policy
442 scheduler_cls: Union[str, Type[object]] = SchedulerConfig.scheduler_cls
443
444 + pooler_config: Optional[PoolerConfig] = ModelConfig.pooler_config
445 override_pooler_config: Optional[Union[dict, PoolerConfig]] = \
446 ModelConfig.override_pooler_config
447 compilation_config: CompilationConfig = \
@@ -579,8 +580,11 @@ class EngineArgs:
580 help=model_kwargs["hf_token"]["help"])
581 model_group.add_argument("--hf-overrides",
582 **model_kwargs["hf_overrides"])
583 + model_group.add_argument("--pooler-config",
584 + **model_kwargs["pooler_config"])
585 model_group.add_argument("--override-pooler-config",
583 - **model_kwargs["override_pooler_config"])
586 + **model_kwargs["override_pooler_config"],
587 + deprecated=True)
588 model_group.add_argument("--logits-processor-pattern",
589 **model_kwargs["logits_processor_pattern"])
590 model_group.add_argument("--generation-config",
@@ -1031,6 +1035,7 @@ class EngineArgs:
1035 mm_shm_cache_max_object_size_mb=self.
1036 mm_shm_cache_max_object_size_mb,
1037 mm_encoder_tp_mode=self.mm_encoder_tp_mode,
1038 + pooler_config=self.pooler_config,
1039 override_pooler_config=self.override_pooler_config,
1040 logits_processor_pattern=self.logits_processor_pattern,
1041 generation_config=self.generation_config,
vllm/entrypoints/llm.py
+7 -3
@@ -151,9 +151,11 @@ class LLM:
151 multi-modal processor obtained from `AutoProcessor.from_pretrained`.
152 The available overrides depend on the model that is being run.
153 For example, for Phi-3-Vision: `{"num_crops": 4}`.
154 - override_pooler_config: Initialize non-default pooling config or
155 - override default pooling config for the pooling model.
156 - e.g. `PoolerConfig(pooling_type="mean", normalize=False)`.
154 + pooler_config: Initialize non-default pooling config for the pooling
155 + model. e.g. `PoolerConfig(pooling_type="mean", normalize=False)`.
156 + override_pooler_config: [DEPRECATED] Use `pooler_config` instead. This
157 + argument is deprecated and will be removed in v0.12.0 or v1.0.0,
158 + whichever is sooner.
159 compilation_config: Either an integer or a dictionary. If it is an
160 integer, it is used as the level of compilation optimization. If it
161 is a dictionary, it can specify the full compilation configuration.
@@ -191,6 +193,7 @@ class LLM:
193 hf_token: Optional[Union[bool, str]] = None,
194 hf_overrides: Optional[HfOverrides] = None,
195 mm_processor_kwargs: Optional[dict[str, Any]] = None,
196 + pooler_config: Optional[PoolerConfig] = None,
197 override_pooler_config: Optional[PoolerConfig] = None,
198 structured_outputs_config: Optional[Union[dict[
199 str, Any], StructuredOutputsConfig]] = None,
@@ -288,6 +291,7 @@ class LLM:
291 hf_token=hf_token,
292 hf_overrides=hf_overrides,
293 mm_processor_kwargs=mm_processor_kwargs,
294 + pooler_config=pooler_config,
295 override_pooler_config=override_pooler_config,
296 structured_outputs_config=structured_outputs_instance,
297 compilation_config=compilation_config_instance,