A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
Adds method to read the pooling types from model's files (#9506)
Signed-off-by: Flavia Beo <flavia.beo@ibm.com> Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Max de Bayser <mbayser@br.ibm.com>
Flávia Béo committed
Nov 7, 2024 at 05:42 UTC
aa9078fa035abfac54179cbdca8b741e49c8cd0b
10 files changed
+342
-25
examples/fp8/quantizer/quantize.py
+2
-2
@@ -230,7 +230,7 @@ def quantize_model(model, quant_cfg, calib_dataloader=None):
230
231
def main(args):
232
if not torch.cuda.is_available():
233
- raise EnvironmentError("GPU is required for inference.")
233
+ raise OSError("GPU is required for inference.")
234
235
random.seed(RAND_SEED)
236
np.random.seed(RAND_SEED)
@@ -314,7 +314,7 @@ def main(args):
314
315
# Workaround for wo quantization
316
if args.qformat in ["int8_wo", "int4_wo", "full_prec"]:
317
- with open(f"{export_path}/config.json", 'r') as f:
317
+ with open(f"{export_path}/config.json") as f:
318
tensorrt_llm_config = json.load(f)
319
if args.qformat == "int8_wo":
320
tensorrt_llm_config["quantization"]["quant_algo"] = 'W8A16'
tests/engine/test_arg_utils.py
+7
@@ -30,6 +30,13 @@ def test_limit_mm_per_prompt_parser(arg, expected):
30
assert args.limit_mm_per_prompt == expected
31
32
33
+def test_valid_pooling_config():
34
+ parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
35
+ args = parser.parse_args(["--pooling-type=MEAN"])
36
+ engine_args = EngineArgs.from_cli_args(args=args)
37
+ assert engine_args.pooling_type == 'MEAN'
38
+
39
+
40
@pytest.mark.parametrize(
41
("arg"),
42
[
tests/model_executor/test_model_load_with_params.py
new
+50
@@ -0,0 +1,50 @@
1
+import os
2
+
3
+import pytest
4
+
5
+from vllm.model_executor.layers.pooler import PoolingType
6
+from vllm.model_executor.models.bert import BertEmbeddingModel
7
+from vllm.platforms import current_platform
8
+
9
+MAX_MODEL_LEN = 128
10
+MODEL_NAME = os.environ.get("MODEL_NAME", "BAAI/bge-base-en-v1.5")
11
+REVISION = os.environ.get("REVISION", "main")
12
+
13
+
14
+@pytest.mark.skipif(current_platform.is_rocm(),
15
+ reason="Xformers backend is not supported on ROCm.")
16
+def test_model_loading_with_params(vllm_runner):
17
+ """
18
+ Test parameter weight loading with tp>1.
19
+ """
20
+ with vllm_runner(model_name=MODEL_NAME,
21
+ revision=REVISION,
22
+ dtype="float16",
23
+ max_model_len=MAX_MODEL_LEN) as model:
24
+ output = model.encode("Write a short story about a robot that"
25
+ " dreams for the first time.\n")
26
+
27
+ model_config = model.model.llm_engine.model_config
28
+
29
+ model_tokenizer = model.model.llm_engine.tokenizer
30
+
31
+ # asserts on the bert model config file
32
+ assert model_config.encoder_config["max_seq_length"] == 512
33
+ assert model_config.encoder_config["do_lower_case"]
34
+
35
+ # asserts on the pooling config files
36
+ assert model_config.pooler_config.pooling_type == PoolingType.CLS.name
37
+ assert model_config.pooler_config.pooling_norm
38
+
39
+ # asserts on the tokenizer loaded
40
+ assert model_tokenizer.tokenizer_id == "BAAI/bge-base-en-v1.5"
41
+ assert model_tokenizer.tokenizer_config["do_lower_case"]
42
+ assert model_tokenizer.tokenizer.model_max_length == 512
43
+
44
+ model = model.model.llm_engine.model_executor\
45
+ .driver_worker.model_runner.model
46
+ assert isinstance(model, BertEmbeddingModel)
47
+ assert model._pooler.pooling_type == PoolingType.CLS
48
+ assert model._pooler.normalize
49
+ # assert output
50
+ assert output
tests/test_config.py
+72
@@ -1,6 +1,8 @@
1
import pytest
2
3
from vllm.config import ModelConfig
4
+from vllm.model_executor.layers.pooler import PoolingType
5
+from vllm.platforms import current_platform
6
7
8
@pytest.mark.parametrize(("model_id", "expected_task"), [
@@ -102,6 +104,76 @@ def test_get_sliding_window():
104
assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
105
106
107
+@pytest.mark.skipif(current_platform.is_rocm(),
108
+ reason="Xformers backend is not supported on ROCm.")
109
+def test_get_pooling_config():
110
+ model_id = "sentence-transformers/all-MiniLM-L12-v2"
111
+ minilm_model_config = ModelConfig(
112
+ model_id,
113
+ task="auto",
114
+ tokenizer=model_id,
115
+ tokenizer_mode="auto",
116
+ trust_remote_code=False,
117
+ seed=0,
118
+ dtype="float16",
119
+ revision=None,
120
+ )
121
+
122
+ minilm_pooling_config = minilm_model_config._init_pooler_config(
123
+ pooling_type=None,
124
+ pooling_norm=None,
125
+ pooling_returned_token_ids=None,
126
+ pooling_softmax=None,
127
+ pooling_step_tag_id=None)
128
+
129
+ assert minilm_pooling_config.pooling_norm
130
+ assert minilm_pooling_config.pooling_type == PoolingType.MEAN.name
131
+
132
+
133
+@pytest.mark.skipif(current_platform.is_rocm(),
134
+ reason="Xformers backend is not supported on ROCm.")
135
+def test_get_pooling_config_from_args():
136
+ model_id = "sentence-transformers/all-MiniLM-L12-v2"
137
+ minilm_model_config = ModelConfig(model_id,
138
+ task="auto",
139
+ tokenizer=model_id,
140
+ tokenizer_mode="auto",
141
+ trust_remote_code=False,
142
+ seed=0,
143
+ dtype="float16",
144
+ revision=None)
145
+
146
+ minilm_pooling_config = minilm_model_config._init_pooler_config(
147
+ pooling_type='CLS',
148
+ pooling_norm=True,
149
+ pooling_returned_token_ids=None,
150
+ pooling_softmax=None,
151
+ pooling_step_tag_id=None)
152
+
153
+ assert minilm_pooling_config.pooling_norm
154
+ assert minilm_pooling_config.pooling_type == PoolingType.CLS.name
155
+
156
+
157
+@pytest.mark.skipif(current_platform.is_rocm(),
158
+ reason="Xformers backend is not supported on ROCm.")
159
+def test_get_bert_tokenization_sentence_transformer_config():
160
+ bge_model_config = ModelConfig(
161
+ model="BAAI/bge-base-en-v1.5",
162
+ task="auto",
163
+ tokenizer="BAAI/bge-base-en-v1.5",
164
+ tokenizer_mode="auto",
165
+ trust_remote_code=False,
166
+ seed=0,
167
+ dtype="float16",
168
+ revision=None,
169
+ )
170
+
171
+ bert_bge_model_config = bge_model_config._get_encoder_config()
172
+
173
+ assert bert_bge_model_config["max_seq_length"] == 512
174
+ assert bert_bge_model_config["do_lower_case"]
175
+
176
+
177
def test_rope_customization():
178
TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
179
TEST_ROPE_THETA = 16_000_000.0
tests/utils.py
+8
-6
@@ -15,6 +15,7 @@ import openai
15
import pytest
16
import requests
17
import torch
18
+import torch.nn.functional as F
19
from openai.types.completion import Completion
20
from typing_extensions import ParamSpec
21
@@ -515,13 +516,14 @@ def compare_all_settings(model: str,
516
ref_result = copy.deepcopy(ref_result)
517
compare_result = copy.deepcopy(compare_result)
518
if "embedding" in ref_result and method == "encode":
518
- ref_embedding = torch.tensor(ref_result["embedding"])
519
- compare_embedding = torch.tensor(
520
- compare_result["embedding"])
521
- mse = ((ref_embedding - compare_embedding)**2).mean()
522
- assert mse < 1e-6, (
519
+ sim = F.cosine_similarity(
520
+ torch.tensor(ref_result["embedding"]),
521
+ torch.tensor(compare_result["embedding"]),
522
+ dim=0,
523
+ )
524
+ assert sim >= 0.999, (
525
f"Embedding for {model=} are not the same.\n"
524
- f"mse={mse}\n")
526
+ f"cosine_similarity={sim}\n")
527
del ref_result["embedding"]
528
del compare_result["embedding"]
529
assert ref_result == compare_result, (
vllm/config.py
+23
-5
@@ -13,10 +13,10 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13
from vllm.model_executor.models import ModelRegistry
14
from vllm.platforms import current_platform
15
from vllm.tracing import is_otel_available, otel_import_error_traceback
16
-from vllm.transformers_utils.config import (ConfigFormat, get_config,
17
- get_hf_image_processor_config,
18
- get_hf_text_config,
19
- is_encoder_decoder, uses_mrope)
16
+from vllm.transformers_utils.config import (
17
+ ConfigFormat, get_config, get_hf_image_processor_config,
18
+ get_hf_text_config, get_pooling_config,
19
+ get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
20
from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
21
print_warning_once)
22
@@ -197,6 +197,7 @@ class ModelConfig:
197
code_revision, rope_scaling, rope_theta,
198
config_format)
199
self.hf_text_config = get_hf_text_config(self.hf_config)
200
+ self.encoder_config = self._get_encoder_config()
201
self.hf_image_processor_config = get_hf_image_processor_config(
202
self.model, revision)
203
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
@@ -229,7 +230,8 @@ class ModelConfig:
230
max_model_len=max_model_len,
231
disable_sliding_window=self.disable_sliding_window,
232
sliding_window_len=self.get_hf_config_sliding_window(),
232
- spec_target_max_model_len=spec_target_max_model_len)
233
+ spec_target_max_model_len=spec_target_max_model_len,
234
+ encoder_config=self.encoder_config)
235
self.served_model_name = get_served_model_name(model,
236
served_model_name)
237
self.multimodal_config = self._init_multimodal_config(
@@ -273,6 +275,10 @@ class ModelConfig:
275
276
return None
277
278
+ def _get_encoder_config(self):
279
+ return get_sentence_transformer_tokenizer_config(
280
+ self.model, self.revision)
281
+
282
def _init_pooler_config(
283
self,
284
pooling_type: Optional[str] = None,
@@ -282,6 +288,14 @@ class ModelConfig:
288
pooling_returned_token_ids: Optional[List[int]] = None
289
) -> Optional["PoolerConfig"]:
290
if self.task == "embedding":
291
+ pooling_config = get_pooling_config(self.model, self.revision)
292
+ if pooling_config is not None:
293
+ # override if user does not
294
+ # specifies pooling_type and/or pooling_norm
295
+ if pooling_type is None:
296
+ pooling_type = pooling_config["pooling_type"]
297
+ if pooling_norm is None:
298
+ pooling_norm = pooling_config["normalize"]
299
return PoolerConfig(
300
pooling_type=pooling_type,
301
pooling_norm=pooling_norm,
@@ -1795,6 +1809,7 @@ def _get_and_verify_max_len(
1809
disable_sliding_window: bool,
1810
sliding_window_len: Optional[Union[int, List[Optional[int]]]],
1811
spec_target_max_model_len: Optional[int] = None,
1812
+ encoder_config: Optional[Any] = None,
1813
) -> int:
1814
"""Get and verify the model's maximum length."""
1815
derived_max_model_len = float("inf")
@@ -1877,6 +1892,9 @@ def _get_and_verify_max_len(
1892
"original_max_position_embeddings"]
1893
derived_max_model_len *= scaling_factor
1894
1895
+ if encoder_config and "max_seq_length" in encoder_config:
1896
+ derived_max_model_len = encoder_config["max_seq_length"]
1897
+
1898
# If the user specified a max length, make sure it is smaller than the
1899
# derived length from the HF model config.
1900
if max_model_len is None:
vllm/engine/arg_utils.py
+2
-1
@@ -16,6 +16,7 @@ from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
16
VllmConfig)
17
from vllm.executor.executor_base import ExecutorBase
18
from vllm.logger import init_logger
19
+from vllm.model_executor.layers.pooler import PoolingType
20
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
21
from vllm.platforms import current_platform
22
from vllm.transformers_utils.config import (
@@ -863,7 +864,7 @@ class EngineArgs:
864
865
parser.add_argument(
866
'--pooling-type',
866
- choices=['LAST', 'ALL', 'CLS', 'STEP'],
867
+ choices=[pt.name for pt in PoolingType],
868
default=None,
869
help='Used to configure the pooling method in the embedding model.'
870
)
vllm/model_executor/layers/pooler.py
+13
-1
@@ -16,6 +16,7 @@ class PoolingType(IntEnum):
16
ALL = 1
17
CLS = 2
18
STEP = 3
19
+ MEAN = 4
20
21
22
class Pooler(nn.Module):
@@ -27,7 +28,7 @@ class Pooler(nn.Module):
28
3. Returns structured results as `PoolerOutput`.
29
30
Attributes:
30
- pooling_type: The type of pooling to use (LAST, ALL, CLS).
31
+ pooling_type: The type of pooling to use.
32
normalize: Whether to normalize the pooled data.
33
"""
34
@@ -97,6 +98,17 @@ class Pooler(nn.Module):
98
for prompt_len in prompt_lens:
99
pooled_data.append(hidden_states[offset:offset + prompt_len])
100
offset += prompt_len
101
+ elif self.pooling_type == PoolingType.MEAN:
102
+ # Calculate mean pooling
103
+ cumsum = torch.cumsum(hidden_states, dim=0)
104
+ start_indices = torch.cat([
105
+ torch.tensor([0], device=hidden_states.device),
106
+ torch.cumsum(prompt_lens[:-1], dim=0)
107
+ ])
108
+ end_indices = torch.cumsum(prompt_lens, dim=0)
109
+ pooled_data = (
110
+ cumsum[end_indices - 1] - cumsum[start_indices] +
111
+ hidden_states[start_indices]) / prompt_lens.unsqueeze(1)
112
elif self.pooling_type == PoolingType.STEP:
113
if self.returned_token_ids is not None and len(
114
self.returned_token_ids) > 0:
vllm/transformers_utils/config.py
+160
-10
@@ -6,6 +6,9 @@ from typing import Any, Dict, Optional, Type, Union
6
import huggingface_hub
7
from huggingface_hub import (file_exists, hf_hub_download,
8
try_to_load_from_cache)
9
+from huggingface_hub.utils import (EntryNotFoundError, LocalEntryNotFoundError,
10
+ RepositoryNotFoundError,
11
+ RevisionNotFoundError)
12
from transformers import GenerationConfig, PretrainedConfig
13
from transformers.models.auto.image_processing_auto import (
14
get_image_processor_config)
@@ -213,7 +216,7 @@ def get_config(
216
raise e
217
218
elif config_format == ConfigFormat.MISTRAL:
216
- config = load_params_config(model, revision)
219
+ config = load_params_config(model, revision, token=kwargs.get("token"))
220
else:
221
raise ValueError(f"Unsupported config format: {config_format}")
222
@@ -243,6 +246,158 @@ def get_config(
246
return config
247
248
249
+def get_hf_file_to_dict(file_name: str,
250
+ model: Union[str, Path],
251
+ revision: Optional[str] = 'main',
252
+ token: Optional[str] = None):
253
+ """
254
+ Downloads a file from the Hugging Face Hub and returns
255
+ its contents as a dictionary.
256
+
257
+ Parameters:
258
+ - file_name (str): The name of the file to download.
259
+ - model (str): The name of the model on the Hugging Face Hub.
260
+ - revision (str): The specific version of the model.
261
+ - token (str): The Hugging Face authentication token.
262
+
263
+ Returns:
264
+ - config_dict (dict): A dictionary containing
265
+ the contents of the downloaded file.
266
+ """
267
+ file_path = Path(model) / file_name
268
+
269
+ if file_or_path_exists(model=model,
270
+ config_name=file_name,
271
+ revision=revision,
272
+ token=token):
273
+
274
+ if not file_path.is_file():
275
+ try:
276
+ hf_hub_file = hf_hub_download(model,
277
+ file_name,
278
+ revision=revision)
279
+ except (RepositoryNotFoundError, RevisionNotFoundError,
280
+ EntryNotFoundError, LocalEntryNotFoundError) as e:
281
+ logger.debug("File or repository not found in hf_hub_download",
282
+ e)
283
+ return None
284
+ file_path = Path(hf_hub_file)
285
+
286
+ with open(file_path) as file:
287
+ return json.load(file)
288
+ return None
289
+
290
+
291
+def get_pooling_config(model: str,
292
+ revision: Optional[str] = 'main',
293
+ token: Optional[str] = None):
294
+ """
295
+ This function gets the pooling and normalize
296
+ config from the model - only applies to
297
+ sentence-transformers models.
298
+
299
+ Args:
300
+ model (str): The name of the Hugging Face model.
301
+ revision (str, optional): The specific version
302
+ of the model to use. Defaults to 'main'.
303
+
304
+ Returns:
305
+ dict: A dictionary containing the pooling
306
+ type and whether normalization is used.
307
+ """
308
+
309
+ modules_file_name = "modules.json"
310
+ modules_dict = get_hf_file_to_dict(modules_file_name, model, revision,
311
+ token)
312
+
313
+ if modules_dict is None:
314
+ return None
315
+
316
+ pooling = next((item for item in modules_dict
317
+ if item["type"] == "sentence_transformers.models.Pooling"),
318
+ None)
319
+ normalize = bool(
320
+ next((item for item in modules_dict
321
+ if item["type"] == "sentence_transformers.models.Normalize"),
322
+ False))
323
+
324
+ if pooling:
325
+
326
+ pooling_file_name = "{}/config.json".format(pooling["path"])
327
+ pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision,
328
+ token)
329
+ pooling_type_name = next(
330
+ (item for item, val in pooling_dict.items() if val is True), None)
331
+
332
+ if pooling_type_name is not None:
333
+ pooling_type_name = get_pooling_config_name(pooling_type_name)
334
+
335
+ return {"pooling_type": pooling_type_name, "normalize": normalize}
336
+
337
+ return None
338
+
339
+
340
+def get_pooling_config_name(pooling_name: str) -> Union[str, None]:
341
+ if "pooling_mode_" in pooling_name:
342
+ pooling_name = pooling_name.replace("pooling_mode_", "")
343
+
344
+ if "_" in pooling_name:
345
+ pooling_name = pooling_name.split("_")[0]
346
+
347
+ if "lasttoken" in pooling_name:
348
+ pooling_name = "last"
349
+
350
+ supported_pooling_types = ['LAST', 'ALL', 'CLS', 'STEP', 'MEAN']
351
+ pooling_type_name = pooling_name.upper()
352
+
353
+ try:
354
+ if pooling_type_name in supported_pooling_types:
355
+ return pooling_type_name
356
+ except NotImplementedError as e:
357
+ logger.debug("Pooling type not supported", e)
358
+ return None
359
+ return None
360
+
361
+
362
+def get_sentence_transformer_tokenizer_config(model: str,
363
+ revision: Optional[str] = 'main',
364
+ token: Optional[str] = None):
365
+ """
366
+ Returns the tokenization configuration dictionary for a
367
+ given Sentence Transformer BERT model.
368
+
369
+ Parameters:
370
+ - model (str): The name of the Sentence Transformer
371
+ BERT model.
372
+ - revision (str, optional): The revision of the m
373
+ odel to use. Defaults to 'main'.
374
+ - token (str): A Hugging Face access token.
375
+
376
+ Returns:
377
+ - dict: A dictionary containing the configuration parameters
378
+ for the Sentence Transformer BERT model.
379
+ """
380
+ for config_name in [
381
+ "sentence_bert_config.json",
382
+ "sentence_roberta_config.json",
383
+ "sentence_distilbert_config.json",
384
+ "sentence_camembert_config.json",
385
+ "sentence_albert_config.json",
386
+ "sentence_xlm-roberta_config.json",
387
+ "sentence_xlnet_config.json",
388
+ ]:
389
+ encoder_dict = get_hf_file_to_dict(config_name, model, revision, token)
390
+ if encoder_dict:
391
+ break
392
+
393
+ if not encoder_dict:
394
+ return None
395
+
396
+ if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
397
+ return encoder_dict
398
+ return None
399
+
400
+
401
def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
402
"""Try to register HF model configuration class to serialize by value
403
@@ -305,20 +460,15 @@ def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
460
exc_info=e)
461
462
308
-def load_params_config(model, revision) -> PretrainedConfig:
463
+def load_params_config(model: Union[str, Path],
464
+ revision: Optional[str],
465
+ token: Optional[str] = None) -> PretrainedConfig:
466
# This function loads a params.json config which
467
# should be used when loading models in mistral format
468
469
config_file_name = "params.json"
470
314
- config_path = Path(model) / config_file_name
315
-
316
- if not config_path.is_file():
317
- config_path = Path(
318
- hf_hub_download(model, config_file_name, revision=revision))
319
-
320
- with open(config_path) as file:
321
- config_dict = json.load(file)
471
+ config_dict = get_hf_file_to_dict(config_file_name, model, revision, token)
472
473
config_mapping = {
474
"dim": "hidden_size",
vllm/transformers_utils/tokenizer_group/__init__.py
+5
@@ -25,6 +25,11 @@ def init_tokenizer_from_configs(model_config: ModelConfig,
25
trust_remote_code=model_config.trust_remote_code,
26
revision=model_config.tokenizer_revision)
27
28
+ if (model_config.encoder_config is not None
29
+ and "do_lower_case" in model_config.encoder_config):
30
+ init_kwargs["do_lower_case"] = model_config.encoder_config[
31
+ "do_lower_case"]
32
+
33
return get_tokenizer_group(parallel_config.tokenizer_pool_config,
34
**init_kwargs)
35