@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ac6d0e75b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Misc] Consolidate pooler config overrides (#10351)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 15, 2024 at 14:59 UTC 2ac6d0e75bc846998da56b50bf4f8853cb36d484
7 files changed +143 -192
docs/source/models/supported_models.rst
+8 -2
@@ -345,6 +345,9 @@ Text Embedding
345 Some model architectures support both generation and embedding tasks.
346 In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
347
348 +.. tip::
349 + You can override the model's pooling method by passing :code:`--override-pooler-config`.
350 +
351 Reward Modeling
352 ---------------
353
@@ -364,7 +367,7 @@ Reward Modeling
367 - ✅︎
368
369 .. note::
367 - As an interim measure, these models are supported via Embeddings API. See `this RFC <https://github.com/vllm-project/vllm/issues/8967>`_ for upcoming changes.
370 + As an interim measure, these models are supported in both offline and online inference via Embeddings API.
371
372 Classification
373 ---------------
@@ -385,7 +388,7 @@ Classification
388 - ✅︎
389
390 .. note::
388 - As an interim measure, these models are supported via Embeddings API. It will be supported via Classification API in the future (no reference APIs exist now).
391 + As an interim measure, these models are supported in both offline and online inference via Embeddings API.
392
393
394 Multimodal Language Models
@@ -600,6 +603,9 @@ Multimodal Embedding
603 Some model architectures support both generation and embedding tasks.
604 In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
605
606 +.. tip::
607 + You can override the model's pooling method by passing :code:`--override-pooler-config`.
608 +
609 Model Support Policy
610 =====================
611
tests/engine/test_arg_utils.py
+7 -2
@@ -2,6 +2,7 @@ from argparse import ArgumentTypeError
2
3 import pytest
4
5 +from vllm.config import PoolerConfig
6 from vllm.engine.arg_utils import EngineArgs, nullable_kvs
7 from vllm.utils import FlexibleArgumentParser
8
@@ -32,9 +33,13 @@ def test_limit_mm_per_prompt_parser(arg, expected):
33
34 def test_valid_pooling_config():
35 parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
35 - args = parser.parse_args(["--pooling-type=MEAN"])
36 + args = parser.parse_args([
37 + '--override-pooler-config',
38 + '{"pooling_type": "MEAN"}',
39 + ])
40 engine_args = EngineArgs.from_cli_args(args=args)
37 - assert engine_args.pooling_type == 'MEAN'
41 + assert engine_args.override_pooler_config == PoolerConfig(
42 + pooling_type="MEAN", )
43
44
45 @pytest.mark.parametrize(
tests/test_config.py
+22 -28
@@ -1,6 +1,8 @@
1 +from dataclasses import asdict
2 +
3 import pytest
4
3 -from vllm.config import ModelConfig
5 +from vllm.config import ModelConfig, PoolerConfig
6 from vllm.model_executor.layers.pooler import PoolingType
7 from vllm.platforms import current_platform
8
@@ -108,7 +110,7 @@ def test_get_sliding_window():
110 reason="Xformers backend is not supported on ROCm.")
111 def test_get_pooling_config():
112 model_id = "sentence-transformers/all-MiniLM-L12-v2"
111 - minilm_model_config = ModelConfig(
113 + model_config = ModelConfig(
114 model_id,
115 task="auto",
116 tokenizer=model_id,
@@ -119,39 +121,31 @@ def test_get_pooling_config():
121 revision=None,
122 )
123
122 - minilm_pooling_config = minilm_model_config._init_pooler_config(
123 - pooling_type=None,
124 - pooling_norm=None,
125 - pooling_returned_token_ids=None,
126 - pooling_softmax=None,
127 - pooling_step_tag_id=None)
124 + pooling_config = model_config._init_pooler_config(None)
125 + assert pooling_config is not None
126
129 - assert minilm_pooling_config.pooling_norm
130 - assert minilm_pooling_config.pooling_type == PoolingType.MEAN.name
127 + assert pooling_config.normalize
128 + assert pooling_config.pooling_type == PoolingType.MEAN.name
129
130
131 @pytest.mark.skipif(current_platform.is_rocm(),
132 reason="Xformers backend is not supported on ROCm.")
133 def test_get_pooling_config_from_args():
134 model_id = "sentence-transformers/all-MiniLM-L12-v2"
137 - minilm_model_config = ModelConfig(model_id,
138 - task="auto",
139 - tokenizer=model_id,
140 - tokenizer_mode="auto",
141 - trust_remote_code=False,
142 - seed=0,
143 - dtype="float16",
144 - revision=None)
145 -
146 - minilm_pooling_config = minilm_model_config._init_pooler_config(
147 - pooling_type='CLS',
148 - pooling_norm=True,
149 - pooling_returned_token_ids=None,
150 - pooling_softmax=None,
151 - pooling_step_tag_id=None)
152 -
153 - assert minilm_pooling_config.pooling_norm
154 - assert minilm_pooling_config.pooling_type == PoolingType.CLS.name
135 + model_config = ModelConfig(model_id,
136 + task="auto",
137 + tokenizer=model_id,
138 + tokenizer_mode="auto",
139 + trust_remote_code=False,
140 + seed=0,
141 + dtype="float16",
142 + revision=None)
143 +
144 + override_config = PoolerConfig(pooling_type='CLS', normalize=True)
145 +
146 + pooling_config = model_config._init_pooler_config(override_config)
147 + assert pooling_config is not None
148 + assert asdict(pooling_config) == asdict(override_config)
149
150
151 @pytest.mark.skipif(current_platform.is_rocm(),
vllm/config.py
+57 -55
@@ -112,10 +112,6 @@ class ModelConfig:
112 the model name will be the same as `model`.
113 limit_mm_per_prompt: Maximum number of data items per modality
114 per prompt. Only applicable for multimodal models.
115 - override_neuron_config: Initialize non default neuron config or
116 - override default neuron config that are specific to Neuron devices,
117 - this argument will be used to configure the neuron config that
118 - can not be gathered from the vllm arguments.
115 config_format: The config format which shall be loaded.
116 Defaults to 'auto' which defaults to 'hf'.
117 hf_overrides: If a dictionary, contains arguments to be forwarded to the
@@ -123,20 +119,12 @@ class ModelConfig:
119 HuggingFace config.
120 mm_processor_kwargs: Arguments to be forwarded to the model's processor
121 for multi-modal data, e.g., image processor.
126 - pooling_type: Used to configure the pooling method in the embedding
127 - model.
128 - pooling_norm: Used to determine whether to normalize the pooled
129 - data in the embedding model.
130 - pooling_softmax: Used to determine whether to softmax the pooled
131 - data in the embedding model.
132 - pooling_step_tag_id: When pooling_step_tag_id is not -1, it indicates
133 - that the score corresponding to the pooling_step_tag_id in the
134 - generated sentence should be returned. Otherwise, it returns
135 - the scores for all tokens.
136 - pooling_returned_token_ids: pooling_returned_token_ids represents a
137 - list of indices for the vocabulary dimensions to be extracted,
138 - such as the token IDs of good_token and bad_token in the
139 - math-shepherd-mistral-7b-prm model.
122 + override_neuron_config: Initialize non default neuron config or
123 + override default neuron config that are specific to Neuron devices,
124 + this argument will be used to configure the neuron config that
125 + can not be gathered from the vllm arguments.
126 + override_pooling_config: Initialize non default pooling config or
127 + override default pooling config for the embedding model.
128 """
129
130 def __init__(
@@ -166,16 +154,12 @@ class ModelConfig:
154 served_model_name: Optional[Union[str, List[str]]] = None,
155 limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
156 use_async_output_proc: bool = True,
169 - override_neuron_config: Optional[Dict[str, Any]] = None,
157 config_format: ConfigFormat = ConfigFormat.AUTO,
158 chat_template_text_format: str = "string",
159 hf_overrides: Optional[HfOverrides] = None,
160 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
174 - pooling_type: Optional[str] = None,
175 - pooling_norm: Optional[bool] = None,
176 - pooling_softmax: Optional[bool] = None,
177 - pooling_step_tag_id: Optional[int] = None,
178 - pooling_returned_token_ids: Optional[List[int]] = None) -> None:
161 + override_neuron_config: Optional[Dict[str, Any]] = None,
162 + override_pooler_config: Optional["PoolerConfig"] = None) -> None:
163 self.model = model
164 self.tokenizer = tokenizer
165 self.tokenizer_mode = tokenizer_mode
@@ -280,13 +264,7 @@ class ModelConfig:
264 supported_tasks, task = self._resolve_task(task, self.hf_config)
265 self.supported_tasks = supported_tasks
266 self.task: Final = task
283 - self.pooler_config = self._init_pooler_config(
284 - pooling_type,
285 - pooling_norm,
286 - pooling_softmax,
287 - pooling_step_tag_id,
288 - pooling_returned_token_ids,
289 - )
267 + self.pooler_config = self._init_pooler_config(override_pooler_config)
268
269 self._verify_quantization()
270 self._verify_cuda_graph()
@@ -311,27 +289,21 @@ class ModelConfig:
289
290 def _init_pooler_config(
291 self,
314 - pooling_type: Optional[str] = None,
315 - pooling_norm: Optional[bool] = None,
316 - pooling_softmax: Optional[bool] = None,
317 - pooling_step_tag_id: Optional[int] = None,
318 - pooling_returned_token_ids: Optional[List[int]] = None
292 + override_pooler_config: Optional["PoolerConfig"],
293 ) -> Optional["PoolerConfig"]:
294 +
295 if self.task == "embedding":
321 - pooling_config = get_pooling_config(self.model, self.revision)
322 - if pooling_config is not None:
323 - # override if user does not
324 - # specifies pooling_type and/or pooling_norm
325 - if pooling_type is None:
326 - pooling_type = pooling_config["pooling_type"]
327 - if pooling_norm is None:
328 - pooling_norm = pooling_config["normalize"]
329 - return PoolerConfig(
330 - pooling_type=pooling_type,
331 - pooling_norm=pooling_norm,
332 - pooling_softmax=pooling_softmax,
333 - pooling_step_tag_id=pooling_step_tag_id,
334 - pooling_returned_token_ids=pooling_returned_token_ids)
296 + user_config = override_pooler_config or PoolerConfig()
297 +
298 + base_config = get_pooling_config(self.model, self.revision)
299 + if base_config is not None:
300 + # Only set values that are not overridden by the user
301 + for k, v in base_config.items():
302 + if getattr(user_config, k) is None:
303 + setattr(user_config, k, v)
304 +
305 + return user_config
306 +
307 return None
308
309 def _init_attention_free(self) -> bool:
@@ -1786,13 +1758,43 @@ class MultiModalConfig:
1758
1759 @dataclass
1760 class PoolerConfig:
1789 - """Controls the behavior of pooler in embedding model"""
1761 + """Controls the behavior of output pooling in embedding models."""
1762
1763 pooling_type: Optional[str] = None
1792 - pooling_norm: Optional[bool] = None
1793 - pooling_softmax: Optional[bool] = None
1794 - pooling_step_tag_id: Optional[int] = None
1795 - pooling_returned_token_ids: Optional[List[int]] = None
1764 + """
1765 + The pooling method of the embedding model. This should be a key in
1766 + :class:`vllm.model_executor.layers.pooler.PoolingType`.
1767 + """
1768 +
1769 + normalize: Optional[bool] = None
1770 + """
1771 + Whether to normalize the pooled outputs. Usually, this should be set to
1772 + ``True`` for embedding outputs.
1773 + """
1774 +
1775 + softmax: Optional[bool] = None
1776 + """
1777 + Whether to apply softmax to the pooled outputs. Usually, this should be set
1778 + to ``True`` for classification outputs.
1779 + """
1780 +
1781 + step_tag_id: Optional[int] = None
1782 + """
1783 + If set, only the score corresponding to the ``step_tag_id`` in the
1784 + generated sentence should be returned. Otherwise, the scores for all tokens
1785 + are returned.
1786 + """
1787 +
1788 + returned_token_ids: Optional[List[int]] = None
1789 + """
1790 + A list of indices for the vocabulary dimensions to be extracted,
1791 + such as the token IDs of ``good_token`` and ``bad_token`` in the
1792 + ``math-shepherd-mistral-7b-prm`` model.
1793 + """
1794 +
1795 + @staticmethod
1796 + def from_json(json_str: str) -> "PoolerConfig":
1797 + return PoolerConfig(**json.loads(json_str))
1798
1799
1800 _STR_DTYPE_TO_TORCH_DTYPE = {
vllm/engine/arg_utils.py
+15 -70
@@ -11,12 +11,11 @@ import vllm.envs as envs
11 from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
12 DeviceConfig, HfOverrides, LoadConfig, LoadFormat,
13 LoRAConfig, ModelConfig, ObservabilityConfig,
14 - ParallelConfig, PromptAdapterConfig, SchedulerConfig,
15 - SpeculativeConfig, TaskOption, TokenizerPoolConfig,
16 - VllmConfig)
14 + ParallelConfig, PoolerConfig, PromptAdapterConfig,
15 + SchedulerConfig, SpeculativeConfig, TaskOption,
16 + TokenizerPoolConfig, VllmConfig)
17 from vllm.executor.executor_base import ExecutorBase
18 from vllm.logger import init_logger
19 -from vllm.model_executor.layers.pooler import PoolingType
19 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
20 from vllm.platforms import current_platform
21 from vllm.transformers_utils.utils import check_gguf_file
@@ -187,15 +186,10 @@ class EngineArgs:
186 otlp_traces_endpoint: Optional[str] = None
187 collect_detailed_traces: Optional[str] = None
188 disable_async_output_proc: bool = False
190 - override_neuron_config: Optional[Dict[str, Any]] = None
189 scheduling_policy: Literal["fcfs", "priority"] = "fcfs"
190
193 - # Pooling configuration.
194 - pooling_type: Optional[str] = None
195 - pooling_norm: Optional[bool] = None
196 - pooling_softmax: Optional[bool] = None
197 - pooling_step_tag_id: Optional[int] = None
198 - pooling_returned_token_ids: Optional[List[int]] = None
191 + override_neuron_config: Optional[Dict[str, Any]] = None
192 + override_pooler_config: Optional[PoolerConfig] = None
193
194 def __post_init__(self):
195 if not self.tokenizer:
@@ -859,12 +853,6 @@ class EngineArgs:
853 default=EngineArgs.disable_async_output_proc,
854 help="Disable async output processing. This may result in "
855 "lower performance.")
862 - parser.add_argument(
863 - '--override-neuron-config',
864 - type=json.loads,
865 - default=None,
866 - help="Override or set neuron device configuration. "
867 - "e.g. {\"cast_logits_dtype\": \"bloat16\"}.'")
856
857 parser.add_argument(
858 '--scheduling-policy',
@@ -877,56 +865,17 @@ class EngineArgs:
865 'arrival deciding any ties).')
866
867 parser.add_argument(
880 - '--pooling-type',
881 - choices=[pt.name for pt in PoolingType],
882 - default=None,
883 - help='Used to configure the pooling method in the embedding model.'
884 - )
885 -
886 - parser.add_argument('--pooling-norm',
887 - default=None,
888 - action='store_true',
889 - help="Used to determine whether to normalize "
890 - "the pooled data in the embedding model.")
891 -
892 - parser.add_argument('--no-pooling-norm',
893 - default=None,
894 - action='store_false',
895 - dest='pooling_norm',
896 - help="Used to determine whether to normalize "
897 - "the pooled data in the embedding model.")
898 -
899 - parser.add_argument('--pooling-softmax',
900 - default=None,
901 - action='store_true',
902 - help="Used to determine whether to softmax "
903 - "the pooled data in the embedding model.")
904 -
905 - parser.add_argument('--no-pooling-softmax',
906 - default=None,
907 - action='store_false',
908 - dest='pooling_softmax',
909 - help="Used to determine whether to softmax "
910 - "the pooled data in the embedding model.")
911 -
912 - parser.add_argument(
913 - '--pooling-step-tag-id',
914 - type=int,
868 + '--override-neuron-config',
869 + type=json.loads,
870 default=None,
916 - help="When pooling-step-tag-id is not -1, it indicates "
917 - "that the score corresponding to the step-tag-ids in the "
918 - "generated sentence should be returned. Otherwise, it "
919 - "returns the scores for all tokens.")
920 -
871 + help="Override or set neuron device configuration. "
872 + "e.g. {\"cast_logits_dtype\": \"bloat16\"}.'")
873 parser.add_argument(
922 - '--pooling-returned-token-ids',
923 - nargs='+',
924 - type=int,
874 + '--override-pooler-config',
875 + type=PoolerConfig.from_json,
876 default=None,
926 - help="pooling-returned-token-ids represents a list of "
927 - "indices for the vocabulary dimensions to be extracted, "
928 - "such as the token IDs of good_token and bad_token in "
929 - "the math-shepherd-mistral-7b-prm model.")
877 + help="Override or set the pooling method in the embedding model. "
878 + "e.g. {\"pooling_type\": \"mean\", \"normalize\": false}.'")
879
880 return parser
881
@@ -967,14 +916,10 @@ class EngineArgs:
916 served_model_name=self.served_model_name,
917 limit_mm_per_prompt=self.limit_mm_per_prompt,
918 use_async_output_proc=not self.disable_async_output_proc,
970 - override_neuron_config=self.override_neuron_config,
919 config_format=self.config_format,
920 mm_processor_kwargs=self.mm_processor_kwargs,
973 - pooling_type=self.pooling_type,
974 - pooling_norm=self.pooling_norm,
975 - pooling_softmax=self.pooling_softmax,
976 - pooling_step_tag_id=self.pooling_step_tag_id,
977 - pooling_returned_token_ids=self.pooling_returned_token_ids,
921 + override_neuron_config=self.override_neuron_config,
922 + override_pooler_config=self.override_pooler_config,
923 )
924
925 def create_load_config(self) -> LoadConfig:
vllm/entrypoints/llm.py
+4 -11
@@ -9,7 +9,8 @@ from tqdm import tqdm
9 from vllm import envs
10 from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
11 BeamSearchSequence, get_beam_search_score)
12 -from vllm.engine.arg_utils import EngineArgs, HfOverrides, TaskOption
12 +from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
13 + TaskOption)
14 from vllm.engine.llm_engine import LLMEngine
15 from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
16 apply_hf_chat_template,
@@ -162,11 +163,7 @@ class LLM:
163 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
164 # After positional args are removed, move this right below `model`
165 task: TaskOption = "auto",
165 - pooling_type: Optional[str] = None,
166 - pooling_norm: Optional[bool] = None,
167 - pooling_softmax: Optional[bool] = None,
168 - pooling_step_tag_id: Optional[int] = None,
169 - pooling_returned_token_ids: Optional[List[int]] = None,
166 + override_pooler_config: Optional[PoolerConfig] = None,
167 **kwargs,
168 ) -> None:
169 '''
@@ -202,11 +199,7 @@ class LLM:
199 disable_async_output_proc=disable_async_output_proc,
200 hf_overrides=hf_overrides,
201 mm_processor_kwargs=mm_processor_kwargs,
205 - pooling_type=pooling_type,
206 - pooling_norm=pooling_norm,
207 - pooling_softmax=pooling_softmax,
208 - pooling_step_tag_id=pooling_step_tag_id,
209 - pooling_returned_token_ids=pooling_returned_token_ids,
202 + override_pooler_config=override_pooler_config,
203 **kwargs,
204 )
205 # Logic to switch between engines is done at runtime instead of import
vllm/model_executor/layers/pooler.py
+30 -24
@@ -63,14 +63,14 @@ class Pooler(nn.Module):
63 return cls(
64 pooling_type=PoolingType[pooler_config.pooling_type]
65 if pooler_config.pooling_type is not None else pooling_type,
66 - normalize=pooler_config.pooling_norm
67 - if pooler_config.pooling_norm is not None else normalize,
68 - softmax=pooler_config.pooling_softmax
69 - if pooler_config.pooling_softmax is not None else softmax,
70 - step_tag_id=pooler_config.pooling_step_tag_id
71 - if pooler_config.pooling_step_tag_id is not None else step_tag_id,
72 - returned_token_ids=pooler_config.pooling_returned_token_ids
73 - if pooler_config.pooling_returned_token_ids is not None else
66 + normalize=pooler_config.normalize
67 + if pooler_config.normalize is not None else normalize,
68 + softmax=pooler_config.softmax
69 + if pooler_config.softmax is not None else softmax,
70 + step_tag_id=pooler_config.step_tag_id
71 + if pooler_config.step_tag_id is not None else step_tag_id,
72 + returned_token_ids=pooler_config.returned_token_ids
73 + if pooler_config.returned_token_ids is not None else
74 returned_token_ids,
75 )
76
@@ -94,10 +94,14 @@ class Pooler(nn.Module):
94 pooled_data = hidden_states[last_token_flat_indices]
95 elif self.pooling_type == PoolingType.ALL:
96 offset = 0
97 - pooled_data = []
97 + pooled_data_lst = []
98 for prompt_len in prompt_lens:
99 - pooled_data.append(hidden_states[offset:offset + prompt_len])
99 + pooled_data_i = hidden_states[offset:offset + prompt_len]
100 +
101 + pooled_data_lst.append(pooled_data_i)
102 offset += prompt_len
103 +
104 + pooled_data = torch.stack(pooled_data_lst)
105 elif self.pooling_type == PoolingType.MEAN:
106 # Calculate mean pooling
107 cumsum = torch.cumsum(hidden_states, dim=0)
@@ -110,24 +114,26 @@ class Pooler(nn.Module):
114 cumsum[end_indices - 1] - cumsum[start_indices] +
115 hidden_states[start_indices]) / prompt_lens.unsqueeze(1)
116 elif self.pooling_type == PoolingType.STEP:
113 - if self.returned_token_ids is not None and len(
114 - self.returned_token_ids) > 0:
115 - logits = hidden_states[:,
116 - self.returned_token_ids].softmax(dim=-1)
117 - else:
118 - logits = hidden_states.softmax(dim=-1)
117 + returned_token_ids = self.returned_token_ids
118 + if returned_token_ids is not None and len(returned_token_ids) > 0:
119 + hidden_states = hidden_states[:, returned_token_ids]
120 +
121 + logits = hidden_states.softmax(dim=-1)
122 + step_tag_id = self.step_tag_id
123 +
124 offset = 0
120 - pooled_data = []
125 + pooled_data_lst = []
126 for prompt_len, seq_data_i in zip(
127 prompt_lens, pooling_metadata.seq_data.values()):
123 - if self.step_tag_id is None:
124 - pooled_data.append(logits[offset:offset + prompt_len])
125 - else:
126 - step_idxs = torch.tensor(
127 - seq_data_i.prompt_token_ids) == self.step_tag_id
128 - pooled_data.append(logits[offset:offset +
129 - prompt_len][step_idxs])
128 + pooled_data_i = logits[offset:offset + prompt_len]
129 + if step_tag_id is not None:
130 + token_ids = torch.tensor(seq_data_i.prompt_token_ids)
131 + pooled_data_i = pooled_data_i[token_ids == step_tag_id]
132 +
133 offset += prompt_len
134 + pooled_data_lst.append(pooled_data_i)
135 +
136 + pooled_data = torch.stack(pooled_data_lst)
137 else:
138 raise ValueError(f"Invalid pooling type: {self.pooling_type}")
139