A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Misc] Consolidate pooler config overrides (#10351)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 15, 2024 at 14:59 UTC
2ac6d0e75bc846998da56b50bf4f8853cb36d484
7 files changed
+143
-192
docs/source/models/supported_models.rst
+8
-2
@@ -345,6 +345,9 @@ Text Embedding
345
Some model architectures support both generation and embedding tasks.
346
In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
347
348
+.. tip::
349
+ You can override the model's pooling method by passing :code:`--override-pooler-config`.
350
+
351
Reward Modeling
352
---------------
353
@@ -364,7 +367,7 @@ Reward Modeling
367
- ✅︎
368
369
.. note::
367
- As an interim measure, these models are supported via Embeddings API. See `this RFC <https://github.com/vllm-project/vllm/issues/8967>`_ for upcoming changes.
370
+ As an interim measure, these models are supported in both offline and online inference via Embeddings API.
371
372
Classification
373
---------------
@@ -385,7 +388,7 @@ Classification
388
- ✅︎
389
390
.. note::
388
- As an interim measure, these models are supported via Embeddings API. It will be supported via Classification API in the future (no reference APIs exist now).
391
+ As an interim measure, these models are supported in both offline and online inference via Embeddings API.
392
393
394
Multimodal Language Models
@@ -600,6 +603,9 @@ Multimodal Embedding
603
Some model architectures support both generation and embedding tasks.
604
In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
605
606
+.. tip::
607
+ You can override the model's pooling method by passing :code:`--override-pooler-config`.
608
+
609
Model Support Policy
610
=====================
611
tests/engine/test_arg_utils.py
+7
-2
@@ -2,6 +2,7 @@ from argparse import ArgumentTypeError
2
3
import pytest
4
5
+from vllm.config import PoolerConfig
6
from vllm.engine.arg_utils import EngineArgs, nullable_kvs
7
from vllm.utils import FlexibleArgumentParser
8
@@ -32,9 +33,13 @@ def test_limit_mm_per_prompt_parser(arg, expected):
33
34
def test_valid_pooling_config():
35
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
35
- args = parser.parse_args(["--pooling-type=MEAN"])
36
+ args = parser.parse_args([
37
+ '--override-pooler-config',
38
+ '{"pooling_type": "MEAN"}',
39
+ ])
40
engine_args = EngineArgs.from_cli_args(args=args)
37
- assert engine_args.pooling_type == 'MEAN'
41
+ assert engine_args.override_pooler_config == PoolerConfig(
42
+ pooling_type="MEAN", )
43
44
45
@pytest.mark.parametrize(
tests/test_config.py
+22
-28
@@ -1,6 +1,8 @@
1
+from dataclasses import asdict
2
+
3
import pytest
4
3
-from vllm.config import ModelConfig
5
+from vllm.config import ModelConfig, PoolerConfig
6
from vllm.model_executor.layers.pooler import PoolingType
7
from vllm.platforms import current_platform
8
@@ -108,7 +110,7 @@ def test_get_sliding_window():
110
reason="Xformers backend is not supported on ROCm.")
111
def test_get_pooling_config():
112
model_id = "sentence-transformers/all-MiniLM-L12-v2"
111
- minilm_model_config = ModelConfig(
113
+ model_config = ModelConfig(
114
model_id,
115
task="auto",
116
tokenizer=model_id,
@@ -119,39 +121,31 @@ def test_get_pooling_config():
121
revision=None,
122
)
123
122
- minilm_pooling_config = minilm_model_config._init_pooler_config(
123
- pooling_type=None,
124
- pooling_norm=None,
125
- pooling_returned_token_ids=None,
126
- pooling_softmax=None,
127
- pooling_step_tag_id=None)
124
+ pooling_config = model_config._init_pooler_config(None)
125
+ assert pooling_config is not None
126
129
- assert minilm_pooling_config.pooling_norm
130
- assert minilm_pooling_config.pooling_type == PoolingType.MEAN.name
127
+ assert pooling_config.normalize
128
+ assert pooling_config.pooling_type == PoolingType.MEAN.name
129
130
131
@pytest.mark.skipif(current_platform.is_rocm(),
132
reason="Xformers backend is not supported on ROCm.")
133
def test_get_pooling_config_from_args():
134
model_id = "sentence-transformers/all-MiniLM-L12-v2"
137
- minilm_model_config = ModelConfig(model_id,
138
- task="auto",
139
- tokenizer=model_id,
140
- tokenizer_mode="auto",
141
- trust_remote_code=False,
142
- seed=0,
143
- dtype="float16",
144
- revision=None)
145
-
146
- minilm_pooling_config = minilm_model_config._init_pooler_config(
147
- pooling_type='CLS',
148
- pooling_norm=True,
149
- pooling_returned_token_ids=None,
150
- pooling_softmax=None,
151
- pooling_step_tag_id=None)
152
-
153
- assert minilm_pooling_config.pooling_norm
154
- assert minilm_pooling_config.pooling_type == PoolingType.CLS.name
135
+ model_config = ModelConfig(model_id,
136
+ task="auto",
137
+ tokenizer=model_id,
138
+ tokenizer_mode="auto",
139
+ trust_remote_code=False,
140
+ seed=0,
141
+ dtype="float16",
142
+ revision=None)
143
+
144
+ override_config = PoolerConfig(pooling_type='CLS', normalize=True)
145
+
146
+ pooling_config = model_config._init_pooler_config(override_config)
147
+ assert pooling_config is not None
148
+ assert asdict(pooling_config) == asdict(override_config)
149
150
151
@pytest.mark.skipif(current_platform.is_rocm(),
vllm/config.py
+57
-55
@@ -112,10 +112,6 @@ class ModelConfig:
112
the model name will be the same as `model`.
113
limit_mm_per_prompt: Maximum number of data items per modality
114
per prompt. Only applicable for multimodal models.
115
- override_neuron_config: Initialize non default neuron config or
116
- override default neuron config that are specific to Neuron devices,
117
- this argument will be used to configure the neuron config that
118
- can not be gathered from the vllm arguments.
115
config_format: The config format which shall be loaded.
116
Defaults to 'auto' which defaults to 'hf'.
117
hf_overrides: If a dictionary, contains arguments to be forwarded to the
@@ -123,20 +119,12 @@ class ModelConfig:
119
HuggingFace config.
120
mm_processor_kwargs: Arguments to be forwarded to the model's processor
121
for multi-modal data, e.g., image processor.
126
- pooling_type: Used to configure the pooling method in the embedding
127
- model.
128
- pooling_norm: Used to determine whether to normalize the pooled
129
- data in the embedding model.
130
- pooling_softmax: Used to determine whether to softmax the pooled
131
- data in the embedding model.
132
- pooling_step_tag_id: When pooling_step_tag_id is not -1, it indicates
133
- that the score corresponding to the pooling_step_tag_id in the
134
- generated sentence should be returned. Otherwise, it returns
135
- the scores for all tokens.
136
- pooling_returned_token_ids: pooling_returned_token_ids represents a
137
- list of indices for the vocabulary dimensions to be extracted,
138
- such as the token IDs of good_token and bad_token in the
139
- math-shepherd-mistral-7b-prm model.
122
+ override_neuron_config: Initialize non default neuron config or
123
+ override default neuron config that are specific to Neuron devices,
124
+ this argument will be used to configure the neuron config that
125
+ can not be gathered from the vllm arguments.
126
+ override_pooling_config: Initialize non default pooling config or
127
+ override default pooling config for the embedding model.
128
"""
129
130
def __init__(
@@ -166,16 +154,12 @@ class ModelConfig:
154
served_model_name: Optional[Union[str, List[str]]] = None,
155
limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
156
use_async_output_proc: bool = True,
169
- override_neuron_config: Optional[Dict[str, Any]] = None,
157
config_format: ConfigFormat = ConfigFormat.AUTO,
158
chat_template_text_format: str = "string",
159
hf_overrides: Optional[HfOverrides] = None,
160
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
174
- pooling_type: Optional[str] = None,
175
- pooling_norm: Optional[bool] = None,
176
- pooling_softmax: Optional[bool] = None,
177
- pooling_step_tag_id: Optional[int] = None,
178
- pooling_returned_token_ids: Optional[List[int]] = None) -> None:
161
+ override_neuron_config: Optional[Dict[str, Any]] = None,
162
+ override_pooler_config: Optional["PoolerConfig"] = None) -> None:
163
self.model = model
164
self.tokenizer = tokenizer
165
self.tokenizer_mode = tokenizer_mode
@@ -280,13 +264,7 @@ class ModelConfig:
264
supported_tasks, task = self._resolve_task(task, self.hf_config)
265
self.supported_tasks = supported_tasks
266
self.task: Final = task
283
- self.pooler_config = self._init_pooler_config(
284
- pooling_type,
285
- pooling_norm,
286
- pooling_softmax,
287
- pooling_step_tag_id,
288
- pooling_returned_token_ids,
289
- )
267
+ self.pooler_config = self._init_pooler_config(override_pooler_config)
268
269
self._verify_quantization()
270
self._verify_cuda_graph()
@@ -311,27 +289,21 @@ class ModelConfig:
289
290
def _init_pooler_config(
291
self,
314
- pooling_type: Optional[str] = None,
315
- pooling_norm: Optional[bool] = None,
316
- pooling_softmax: Optional[bool] = None,
317
- pooling_step_tag_id: Optional[int] = None,
318
- pooling_returned_token_ids: Optional[List[int]] = None
292
+ override_pooler_config: Optional["PoolerConfig"],
293
) -> Optional["PoolerConfig"]:
294
+
295
if self.task == "embedding":
321
- pooling_config = get_pooling_config(self.model, self.revision)
322
- if pooling_config is not None:
323
- # override if user does not
324
- # specifies pooling_type and/or pooling_norm
325
- if pooling_type is None:
326
- pooling_type = pooling_config["pooling_type"]
327
- if pooling_norm is None:
328
- pooling_norm = pooling_config["normalize"]
329
- return PoolerConfig(
330
- pooling_type=pooling_type,
331
- pooling_norm=pooling_norm,
332
- pooling_softmax=pooling_softmax,
333
- pooling_step_tag_id=pooling_step_tag_id,
334
- pooling_returned_token_ids=pooling_returned_token_ids)
296
+ user_config = override_pooler_config or PoolerConfig()
297
+
298
+ base_config = get_pooling_config(self.model, self.revision)
299
+ if base_config is not None:
300
+ # Only set values that are not overridden by the user
301
+ for k, v in base_config.items():
302
+ if getattr(user_config, k) is None:
303
+ setattr(user_config, k, v)
304
+
305
+ return user_config
306
+
307
return None
308
309
def _init_attention_free(self) -> bool:
@@ -1786,13 +1758,43 @@ class MultiModalConfig:
1758
1759
@dataclass
1760
class PoolerConfig:
1789
- """Controls the behavior of pooler in embedding model"""
1761
+ """Controls the behavior of output pooling in embedding models."""
1762
1763
pooling_type: Optional[str] = None
1792
- pooling_norm: Optional[bool] = None
1793
- pooling_softmax: Optional[bool] = None
1794
- pooling_step_tag_id: Optional[int] = None
1795
- pooling_returned_token_ids: Optional[List[int]] = None
1764
+ """
1765
+ The pooling method of the embedding model. This should be a key in
1766
+ :class:`vllm.model_executor.layers.pooler.PoolingType`.
1767
+ """
1768
+
1769
+ normalize: Optional[bool] = None
1770
+ """
1771
+ Whether to normalize the pooled outputs. Usually, this should be set to
1772
+ ``True`` for embedding outputs.
1773
+ """
1774
+
1775
+ softmax: Optional[bool] = None
1776
+ """
1777
+ Whether to apply softmax to the pooled outputs. Usually, this should be set
1778
+ to ``True`` for classification outputs.
1779
+ """
1780
+
1781
+ step_tag_id: Optional[int] = None
1782
+ """
1783
+ If set, only the score corresponding to the ``step_tag_id`` in the
1784
+ generated sentence should be returned. Otherwise, the scores for all tokens
1785
+ are returned.
1786
+ """
1787
+
1788
+ returned_token_ids: Optional[List[int]] = None
1789
+ """
1790
+ A list of indices for the vocabulary dimensions to be extracted,
1791
+ such as the token IDs of ``good_token`` and ``bad_token`` in the
1792
+ ``math-shepherd-mistral-7b-prm`` model.
1793
+ """
1794
+
1795
+ @staticmethod
1796
+ def from_json(json_str: str) -> "PoolerConfig":
1797
+ return PoolerConfig(**json.loads(json_str))
1798
1799
1800
_STR_DTYPE_TO_TORCH_DTYPE = {
vllm/engine/arg_utils.py
+15
-70
@@ -11,12 +11,11 @@ import vllm.envs as envs
11
from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
12
DeviceConfig, HfOverrides, LoadConfig, LoadFormat,
13
LoRAConfig, ModelConfig, ObservabilityConfig,
14
- ParallelConfig, PromptAdapterConfig, SchedulerConfig,
15
- SpeculativeConfig, TaskOption, TokenizerPoolConfig,
16
- VllmConfig)
14
+ ParallelConfig, PoolerConfig, PromptAdapterConfig,
15
+ SchedulerConfig, SpeculativeConfig, TaskOption,
16
+ TokenizerPoolConfig, VllmConfig)
17
from vllm.executor.executor_base import ExecutorBase
18
from vllm.logger import init_logger
19
-from vllm.model_executor.layers.pooler import PoolingType
19
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
20
from vllm.platforms import current_platform
21
from vllm.transformers_utils.utils import check_gguf_file
@@ -187,15 +186,10 @@ class EngineArgs:
186
otlp_traces_endpoint: Optional[str] = None
187
collect_detailed_traces: Optional[str] = None
188
disable_async_output_proc: bool = False
190
- override_neuron_config: Optional[Dict[str, Any]] = None
189
scheduling_policy: Literal["fcfs", "priority"] = "fcfs"
190
193
- # Pooling configuration.
194
- pooling_type: Optional[str] = None
195
- pooling_norm: Optional[bool] = None
196
- pooling_softmax: Optional[bool] = None
197
- pooling_step_tag_id: Optional[int] = None
198
- pooling_returned_token_ids: Optional[List[int]] = None
191
+ override_neuron_config: Optional[Dict[str, Any]] = None
192
+ override_pooler_config: Optional[PoolerConfig] = None
193
194
def __post_init__(self):
195
if not self.tokenizer:
@@ -859,12 +853,6 @@ class EngineArgs:
853
default=EngineArgs.disable_async_output_proc,
854
help="Disable async output processing. This may result in "
855
"lower performance.")
862
- parser.add_argument(
863
- '--override-neuron-config',
864
- type=json.loads,
865
- default=None,
866
- help="Override or set neuron device configuration. "
867
- "e.g. {\"cast_logits_dtype\": \"bloat16\"}.'")
856
857
parser.add_argument(
858
'--scheduling-policy',
@@ -877,56 +865,17 @@ class EngineArgs:
865
'arrival deciding any ties).')
866
867
parser.add_argument(
880
- '--pooling-type',
881
- choices=[pt.name for pt in PoolingType],
882
- default=None,
883
- help='Used to configure the pooling method in the embedding model.'
884
- )
885
-
886
- parser.add_argument('--pooling-norm',
887
- default=None,
888
- action='store_true',
889
- help="Used to determine whether to normalize "
890
- "the pooled data in the embedding model.")
891
-
892
- parser.add_argument('--no-pooling-norm',
893
- default=None,
894
- action='store_false',
895
- dest='pooling_norm',
896
- help="Used to determine whether to normalize "
897
- "the pooled data in the embedding model.")
898
-
899
- parser.add_argument('--pooling-softmax',
900
- default=None,
901
- action='store_true',
902
- help="Used to determine whether to softmax "
903
- "the pooled data in the embedding model.")
904
-
905
- parser.add_argument('--no-pooling-softmax',
906
- default=None,
907
- action='store_false',
908
- dest='pooling_softmax',
909
- help="Used to determine whether to softmax "
910
- "the pooled data in the embedding model.")
911
-
912
- parser.add_argument(
913
- '--pooling-step-tag-id',
914
- type=int,
868
+ '--override-neuron-config',
869
+ type=json.loads,
870
default=None,
916
- help="When pooling-step-tag-id is not -1, it indicates "
917
- "that the score corresponding to the step-tag-ids in the "
918
- "generated sentence should be returned. Otherwise, it "
919
- "returns the scores for all tokens.")
920
-
871
+ help="Override or set neuron device configuration. "
872
+ "e.g. {\"cast_logits_dtype\": \"bloat16\"}.'")
873
parser.add_argument(
922
- '--pooling-returned-token-ids',
923
- nargs='+',
924
- type=int,
874
+ '--override-pooler-config',
875
+ type=PoolerConfig.from_json,
876
default=None,
926
- help="pooling-returned-token-ids represents a list of "
927
- "indices for the vocabulary dimensions to be extracted, "
928
- "such as the token IDs of good_token and bad_token in "
929
- "the math-shepherd-mistral-7b-prm model.")
877
+ help="Override or set the pooling method in the embedding model. "
878
+ "e.g. {\"pooling_type\": \"mean\", \"normalize\": false}.'")
879
880
return parser
881
@@ -967,14 +916,10 @@ class EngineArgs:
916
served_model_name=self.served_model_name,
917
limit_mm_per_prompt=self.limit_mm_per_prompt,
918
use_async_output_proc=not self.disable_async_output_proc,
970
- override_neuron_config=self.override_neuron_config,
919
config_format=self.config_format,
920
mm_processor_kwargs=self.mm_processor_kwargs,
973
- pooling_type=self.pooling_type,
974
- pooling_norm=self.pooling_norm,
975
- pooling_softmax=self.pooling_softmax,
976
- pooling_step_tag_id=self.pooling_step_tag_id,
977
- pooling_returned_token_ids=self.pooling_returned_token_ids,
921
+ override_neuron_config=self.override_neuron_config,
922
+ override_pooler_config=self.override_pooler_config,
923
)
924
925
def create_load_config(self) -> LoadConfig:
vllm/entrypoints/llm.py
+4
-11
@@ -9,7 +9,8 @@ from tqdm import tqdm
9
from vllm import envs
10
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
11
BeamSearchSequence, get_beam_search_score)
12
-from vllm.engine.arg_utils import EngineArgs, HfOverrides, TaskOption
12
+from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
13
+ TaskOption)
14
from vllm.engine.llm_engine import LLMEngine
15
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
16
apply_hf_chat_template,
@@ -162,11 +163,7 @@ class LLM:
163
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
164
# After positional args are removed, move this right below `model`
165
task: TaskOption = "auto",
165
- pooling_type: Optional[str] = None,
166
- pooling_norm: Optional[bool] = None,
167
- pooling_softmax: Optional[bool] = None,
168
- pooling_step_tag_id: Optional[int] = None,
169
- pooling_returned_token_ids: Optional[List[int]] = None,
166
+ override_pooler_config: Optional[PoolerConfig] = None,
167
**kwargs,
168
) -> None:
169
'''
@@ -202,11 +199,7 @@ class LLM:
199
disable_async_output_proc=disable_async_output_proc,
200
hf_overrides=hf_overrides,
201
mm_processor_kwargs=mm_processor_kwargs,
205
- pooling_type=pooling_type,
206
- pooling_norm=pooling_norm,
207
- pooling_softmax=pooling_softmax,
208
- pooling_step_tag_id=pooling_step_tag_id,
209
- pooling_returned_token_ids=pooling_returned_token_ids,
202
+ override_pooler_config=override_pooler_config,
203
**kwargs,
204
)
205
# Logic to switch between engines is done at runtime instead of import
vllm/model_executor/layers/pooler.py
+30
-24
@@ -63,14 +63,14 @@ class Pooler(nn.Module):
63
return cls(
64
pooling_type=PoolingType[pooler_config.pooling_type]
65
if pooler_config.pooling_type is not None else pooling_type,
66
- normalize=pooler_config.pooling_norm
67
- if pooler_config.pooling_norm is not None else normalize,
68
- softmax=pooler_config.pooling_softmax
69
- if pooler_config.pooling_softmax is not None else softmax,
70
- step_tag_id=pooler_config.pooling_step_tag_id
71
- if pooler_config.pooling_step_tag_id is not None else step_tag_id,
72
- returned_token_ids=pooler_config.pooling_returned_token_ids
73
- if pooler_config.pooling_returned_token_ids is not None else
66
+ normalize=pooler_config.normalize
67
+ if pooler_config.normalize is not None else normalize,
68
+ softmax=pooler_config.softmax
69
+ if pooler_config.softmax is not None else softmax,
70
+ step_tag_id=pooler_config.step_tag_id
71
+ if pooler_config.step_tag_id is not None else step_tag_id,
72
+ returned_token_ids=pooler_config.returned_token_ids
73
+ if pooler_config.returned_token_ids is not None else
74
returned_token_ids,
75
)
76
@@ -94,10 +94,14 @@ class Pooler(nn.Module):
94
pooled_data = hidden_states[last_token_flat_indices]
95
elif self.pooling_type == PoolingType.ALL:
96
offset = 0
97
- pooled_data = []
97
+ pooled_data_lst = []
98
for prompt_len in prompt_lens:
99
- pooled_data.append(hidden_states[offset:offset + prompt_len])
99
+ pooled_data_i = hidden_states[offset:offset + prompt_len]
100
+
101
+ pooled_data_lst.append(pooled_data_i)
102
offset += prompt_len
103
+
104
+ pooled_data = torch.stack(pooled_data_lst)
105
elif self.pooling_type == PoolingType.MEAN:
106
# Calculate mean pooling
107
cumsum = torch.cumsum(hidden_states, dim=0)
@@ -110,24 +114,26 @@ class Pooler(nn.Module):
114
cumsum[end_indices - 1] - cumsum[start_indices] +
115
hidden_states[start_indices]) / prompt_lens.unsqueeze(1)
116
elif self.pooling_type == PoolingType.STEP:
113
- if self.returned_token_ids is not None and len(
114
- self.returned_token_ids) > 0:
115
- logits = hidden_states[:,
116
- self.returned_token_ids].softmax(dim=-1)
117
- else:
118
- logits = hidden_states.softmax(dim=-1)
117
+ returned_token_ids = self.returned_token_ids
118
+ if returned_token_ids is not None and len(returned_token_ids) > 0:
119
+ hidden_states = hidden_states[:, returned_token_ids]
120
+
121
+ logits = hidden_states.softmax(dim=-1)
122
+ step_tag_id = self.step_tag_id
123
+
124
offset = 0
120
- pooled_data = []
125
+ pooled_data_lst = []
126
for prompt_len, seq_data_i in zip(
127
prompt_lens, pooling_metadata.seq_data.values()):
123
- if self.step_tag_id is None:
124
- pooled_data.append(logits[offset:offset + prompt_len])
125
- else:
126
- step_idxs = torch.tensor(
127
- seq_data_i.prompt_token_ids) == self.step_tag_id
128
- pooled_data.append(logits[offset:offset +
129
- prompt_len][step_idxs])
128
+ pooled_data_i = logits[offset:offset + prompt_len]
129
+ if step_tag_id is not None:
130
+ token_ids = torch.tensor(seq_data_i.prompt_token_ids)
131
+ pooled_data_i = pooled_data_i[token_ids == step_tag_id]
132
+
133
offset += prompt_len
134
+ pooled_data_lst.append(pooled_data_i)
135
+
136
+ pooled_data = torch.stack(pooled_data_lst)
137
else:
138
raise ValueError(f"Invalid pooling type: {self.pooling_type}")
139