@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aa9078fa03
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

Adds method to read the pooling types from model's files (#9506)

Signed-off-by: Flavia Beo <flavia.beo@ibm.com> Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Max de Bayser <mbayser@br.ibm.com>

Flávia Béo committed Nov 7, 2024 at 05:42 UTC aa9078fa035abfac54179cbdca8b741e49c8cd0b
10 files changed +342 -25
examples/fp8/quantizer/quantize.py
+2 -2
@@ -230,7 +230,7 @@ def quantize_model(model, quant_cfg, calib_dataloader=None):
230
231 def main(args):
232 if not torch.cuda.is_available():
233 - raise EnvironmentError("GPU is required for inference.")
233 + raise OSError("GPU is required for inference.")
234
235 random.seed(RAND_SEED)
236 np.random.seed(RAND_SEED)
@@ -314,7 +314,7 @@ def main(args):
314
315 # Workaround for wo quantization
316 if args.qformat in ["int8_wo", "int4_wo", "full_prec"]:
317 - with open(f"{export_path}/config.json", 'r') as f:
317 + with open(f"{export_path}/config.json") as f:
318 tensorrt_llm_config = json.load(f)
319 if args.qformat == "int8_wo":
320 tensorrt_llm_config["quantization"]["quant_algo"] = 'W8A16'
tests/engine/test_arg_utils.py
+7
@@ -30,6 +30,13 @@ def test_limit_mm_per_prompt_parser(arg, expected):
30 assert args.limit_mm_per_prompt == expected
31
32
33 +def test_valid_pooling_config():
34 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
35 + args = parser.parse_args(["--pooling-type=MEAN"])
36 + engine_args = EngineArgs.from_cli_args(args=args)
37 + assert engine_args.pooling_type == 'MEAN'
38 +
39 +
40 @pytest.mark.parametrize(
41 ("arg"),
42 [
tests/model_executor/test_model_load_with_params.py new
+50
@@ -0,0 +1,50 @@
1 +import os
2 +
3 +import pytest
4 +
5 +from vllm.model_executor.layers.pooler import PoolingType
6 +from vllm.model_executor.models.bert import BertEmbeddingModel
7 +from vllm.platforms import current_platform
8 +
9 +MAX_MODEL_LEN = 128
10 +MODEL_NAME = os.environ.get("MODEL_NAME", "BAAI/bge-base-en-v1.5")
11 +REVISION = os.environ.get("REVISION", "main")
12 +
13 +
14 +@pytest.mark.skipif(current_platform.is_rocm(),
15 + reason="Xformers backend is not supported on ROCm.")
16 +def test_model_loading_with_params(vllm_runner):
17 + """
18 + Test parameter weight loading with tp>1.
19 + """
20 + with vllm_runner(model_name=MODEL_NAME,
21 + revision=REVISION,
22 + dtype="float16",
23 + max_model_len=MAX_MODEL_LEN) as model:
24 + output = model.encode("Write a short story about a robot that"
25 + " dreams for the first time.\n")
26 +
27 + model_config = model.model.llm_engine.model_config
28 +
29 + model_tokenizer = model.model.llm_engine.tokenizer
30 +
31 + # asserts on the bert model config file
32 + assert model_config.encoder_config["max_seq_length"] == 512
33 + assert model_config.encoder_config["do_lower_case"]
34 +
35 + # asserts on the pooling config files
36 + assert model_config.pooler_config.pooling_type == PoolingType.CLS.name
37 + assert model_config.pooler_config.pooling_norm
38 +
39 + # asserts on the tokenizer loaded
40 + assert model_tokenizer.tokenizer_id == "BAAI/bge-base-en-v1.5"
41 + assert model_tokenizer.tokenizer_config["do_lower_case"]
42 + assert model_tokenizer.tokenizer.model_max_length == 512
43 +
44 + model = model.model.llm_engine.model_executor\
45 + .driver_worker.model_runner.model
46 + assert isinstance(model, BertEmbeddingModel)
47 + assert model._pooler.pooling_type == PoolingType.CLS
48 + assert model._pooler.normalize
49 + # assert output
50 + assert output
tests/test_config.py
+72
@@ -1,6 +1,8 @@
1 import pytest
2
3 from vllm.config import ModelConfig
4 +from vllm.model_executor.layers.pooler import PoolingType
5 +from vllm.platforms import current_platform
6
7
8 @pytest.mark.parametrize(("model_id", "expected_task"), [
@@ -102,6 +104,76 @@ def test_get_sliding_window():
104 assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
105
106
107 +@pytest.mark.skipif(current_platform.is_rocm(),
108 + reason="Xformers backend is not supported on ROCm.")
109 +def test_get_pooling_config():
110 + model_id = "sentence-transformers/all-MiniLM-L12-v2"
111 + minilm_model_config = ModelConfig(
112 + model_id,
113 + task="auto",
114 + tokenizer=model_id,
115 + tokenizer_mode="auto",
116 + trust_remote_code=False,
117 + seed=0,
118 + dtype="float16",
119 + revision=None,
120 + )
121 +
122 + minilm_pooling_config = minilm_model_config._init_pooler_config(
123 + pooling_type=None,
124 + pooling_norm=None,
125 + pooling_returned_token_ids=None,
126 + pooling_softmax=None,
127 + pooling_step_tag_id=None)
128 +
129 + assert minilm_pooling_config.pooling_norm
130 + assert minilm_pooling_config.pooling_type == PoolingType.MEAN.name
131 +
132 +
133 +@pytest.mark.skipif(current_platform.is_rocm(),
134 + reason="Xformers backend is not supported on ROCm.")
135 +def test_get_pooling_config_from_args():
136 + model_id = "sentence-transformers/all-MiniLM-L12-v2"
137 + minilm_model_config = ModelConfig(model_id,
138 + task="auto",
139 + tokenizer=model_id,
140 + tokenizer_mode="auto",
141 + trust_remote_code=False,
142 + seed=0,
143 + dtype="float16",
144 + revision=None)
145 +
146 + minilm_pooling_config = minilm_model_config._init_pooler_config(
147 + pooling_type='CLS',
148 + pooling_norm=True,
149 + pooling_returned_token_ids=None,
150 + pooling_softmax=None,
151 + pooling_step_tag_id=None)
152 +
153 + assert minilm_pooling_config.pooling_norm
154 + assert minilm_pooling_config.pooling_type == PoolingType.CLS.name
155 +
156 +
157 +@pytest.mark.skipif(current_platform.is_rocm(),
158 + reason="Xformers backend is not supported on ROCm.")
159 +def test_get_bert_tokenization_sentence_transformer_config():
160 + bge_model_config = ModelConfig(
161 + model="BAAI/bge-base-en-v1.5",
162 + task="auto",
163 + tokenizer="BAAI/bge-base-en-v1.5",
164 + tokenizer_mode="auto",
165 + trust_remote_code=False,
166 + seed=0,
167 + dtype="float16",
168 + revision=None,
169 + )
170 +
171 + bert_bge_model_config = bge_model_config._get_encoder_config()
172 +
173 + assert bert_bge_model_config["max_seq_length"] == 512
174 + assert bert_bge_model_config["do_lower_case"]
175 +
176 +
177 def test_rope_customization():
178 TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
179 TEST_ROPE_THETA = 16_000_000.0
tests/utils.py
+8 -6
@@ -15,6 +15,7 @@ import openai
15 import pytest
16 import requests
17 import torch
18 +import torch.nn.functional as F
19 from openai.types.completion import Completion
20 from typing_extensions import ParamSpec
21
@@ -515,13 +516,14 @@ def compare_all_settings(model: str,
516 ref_result = copy.deepcopy(ref_result)
517 compare_result = copy.deepcopy(compare_result)
518 if "embedding" in ref_result and method == "encode":
518 - ref_embedding = torch.tensor(ref_result["embedding"])
519 - compare_embedding = torch.tensor(
520 - compare_result["embedding"])
521 - mse = ((ref_embedding - compare_embedding)**2).mean()
522 - assert mse < 1e-6, (
519 + sim = F.cosine_similarity(
520 + torch.tensor(ref_result["embedding"]),
521 + torch.tensor(compare_result["embedding"]),
522 + dim=0,
523 + )
524 + assert sim >= 0.999, (
525 f"Embedding for {model=} are not the same.\n"
524 - f"mse={mse}\n")
526 + f"cosine_similarity={sim}\n")
527 del ref_result["embedding"]
528 del compare_result["embedding"]
529 assert ref_result == compare_result, (
vllm/config.py
+23 -5
@@ -13,10 +13,10 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13 from vllm.model_executor.models import ModelRegistry
14 from vllm.platforms import current_platform
15 from vllm.tracing import is_otel_available, otel_import_error_traceback
16 -from vllm.transformers_utils.config import (ConfigFormat, get_config,
17 - get_hf_image_processor_config,
18 - get_hf_text_config,
19 - is_encoder_decoder, uses_mrope)
16 +from vllm.transformers_utils.config import (
17 + ConfigFormat, get_config, get_hf_image_processor_config,
18 + get_hf_text_config, get_pooling_config,
19 + get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
20 from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
21 print_warning_once)
22
@@ -197,6 +197,7 @@ class ModelConfig:
197 code_revision, rope_scaling, rope_theta,
198 config_format)
199 self.hf_text_config = get_hf_text_config(self.hf_config)
200 + self.encoder_config = self._get_encoder_config()
201 self.hf_image_processor_config = get_hf_image_processor_config(
202 self.model, revision)
203 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
@@ -229,7 +230,8 @@ class ModelConfig:
230 max_model_len=max_model_len,
231 disable_sliding_window=self.disable_sliding_window,
232 sliding_window_len=self.get_hf_config_sliding_window(),
232 - spec_target_max_model_len=spec_target_max_model_len)
233 + spec_target_max_model_len=spec_target_max_model_len,
234 + encoder_config=self.encoder_config)
235 self.served_model_name = get_served_model_name(model,
236 served_model_name)
237 self.multimodal_config = self._init_multimodal_config(
@@ -273,6 +275,10 @@ class ModelConfig:
275
276 return None
277
278 + def _get_encoder_config(self):
279 + return get_sentence_transformer_tokenizer_config(
280 + self.model, self.revision)
281 +
282 def _init_pooler_config(
283 self,
284 pooling_type: Optional[str] = None,
@@ -282,6 +288,14 @@ class ModelConfig:
288 pooling_returned_token_ids: Optional[List[int]] = None
289 ) -> Optional["PoolerConfig"]:
290 if self.task == "embedding":
291 + pooling_config = get_pooling_config(self.model, self.revision)
292 + if pooling_config is not None:
293 + # override if user does not
294 + # specifies pooling_type and/or pooling_norm
295 + if pooling_type is None:
296 + pooling_type = pooling_config["pooling_type"]
297 + if pooling_norm is None:
298 + pooling_norm = pooling_config["normalize"]
299 return PoolerConfig(
300 pooling_type=pooling_type,
301 pooling_norm=pooling_norm,
@@ -1795,6 +1809,7 @@ def _get_and_verify_max_len(
1809 disable_sliding_window: bool,
1810 sliding_window_len: Optional[Union[int, List[Optional[int]]]],
1811 spec_target_max_model_len: Optional[int] = None,
1812 + encoder_config: Optional[Any] = None,
1813 ) -> int:
1814 """Get and verify the model's maximum length."""
1815 derived_max_model_len = float("inf")
@@ -1877,6 +1892,9 @@ def _get_and_verify_max_len(
1892 "original_max_position_embeddings"]
1893 derived_max_model_len *= scaling_factor
1894
1895 + if encoder_config and "max_seq_length" in encoder_config:
1896 + derived_max_model_len = encoder_config["max_seq_length"]
1897 +
1898 # If the user specified a max length, make sure it is smaller than the
1899 # derived length from the HF model config.
1900 if max_model_len is None:
vllm/engine/arg_utils.py
+2 -1
@@ -16,6 +16,7 @@ from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
16 VllmConfig)
17 from vllm.executor.executor_base import ExecutorBase
18 from vllm.logger import init_logger
19 +from vllm.model_executor.layers.pooler import PoolingType
20 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
21 from vllm.platforms import current_platform
22 from vllm.transformers_utils.config import (
@@ -863,7 +864,7 @@ class EngineArgs:
864
865 parser.add_argument(
866 '--pooling-type',
866 - choices=['LAST', 'ALL', 'CLS', 'STEP'],
867 + choices=[pt.name for pt in PoolingType],
868 default=None,
869 help='Used to configure the pooling method in the embedding model.'
870 )
vllm/model_executor/layers/pooler.py
+13 -1
@@ -16,6 +16,7 @@ class PoolingType(IntEnum):
16 ALL = 1
17 CLS = 2
18 STEP = 3
19 + MEAN = 4
20
21
22 class Pooler(nn.Module):
@@ -27,7 +28,7 @@ class Pooler(nn.Module):
28 3. Returns structured results as `PoolerOutput`.
29
30 Attributes:
30 - pooling_type: The type of pooling to use (LAST, ALL, CLS).
31 + pooling_type: The type of pooling to use.
32 normalize: Whether to normalize the pooled data.
33 """
34
@@ -97,6 +98,17 @@ class Pooler(nn.Module):
98 for prompt_len in prompt_lens:
99 pooled_data.append(hidden_states[offset:offset + prompt_len])
100 offset += prompt_len
101 + elif self.pooling_type == PoolingType.MEAN:
102 + # Calculate mean pooling
103 + cumsum = torch.cumsum(hidden_states, dim=0)
104 + start_indices = torch.cat([
105 + torch.tensor([0], device=hidden_states.device),
106 + torch.cumsum(prompt_lens[:-1], dim=0)
107 + ])
108 + end_indices = torch.cumsum(prompt_lens, dim=0)
109 + pooled_data = (
110 + cumsum[end_indices - 1] - cumsum[start_indices] +
111 + hidden_states[start_indices]) / prompt_lens.unsqueeze(1)
112 elif self.pooling_type == PoolingType.STEP:
113 if self.returned_token_ids is not None and len(
114 self.returned_token_ids) > 0:
vllm/transformers_utils/config.py
+160 -10
@@ -6,6 +6,9 @@ from typing import Any, Dict, Optional, Type, Union
6 import huggingface_hub
7 from huggingface_hub import (file_exists, hf_hub_download,
8 try_to_load_from_cache)
9 +from huggingface_hub.utils import (EntryNotFoundError, LocalEntryNotFoundError,
10 + RepositoryNotFoundError,
11 + RevisionNotFoundError)
12 from transformers import GenerationConfig, PretrainedConfig
13 from transformers.models.auto.image_processing_auto import (
14 get_image_processor_config)
@@ -213,7 +216,7 @@ def get_config(
216 raise e
217
218 elif config_format == ConfigFormat.MISTRAL:
216 - config = load_params_config(model, revision)
219 + config = load_params_config(model, revision, token=kwargs.get("token"))
220 else:
221 raise ValueError(f"Unsupported config format: {config_format}")
222
@@ -243,6 +246,158 @@ def get_config(
246 return config
247
248
249 +def get_hf_file_to_dict(file_name: str,
250 + model: Union[str, Path],
251 + revision: Optional[str] = 'main',
252 + token: Optional[str] = None):
253 + """
254 + Downloads a file from the Hugging Face Hub and returns
255 + its contents as a dictionary.
256 +
257 + Parameters:
258 + - file_name (str): The name of the file to download.
259 + - model (str): The name of the model on the Hugging Face Hub.
260 + - revision (str): The specific version of the model.
261 + - token (str): The Hugging Face authentication token.
262 +
263 + Returns:
264 + - config_dict (dict): A dictionary containing
265 + the contents of the downloaded file.
266 + """
267 + file_path = Path(model) / file_name
268 +
269 + if file_or_path_exists(model=model,
270 + config_name=file_name,
271 + revision=revision,
272 + token=token):
273 +
274 + if not file_path.is_file():
275 + try:
276 + hf_hub_file = hf_hub_download(model,
277 + file_name,
278 + revision=revision)
279 + except (RepositoryNotFoundError, RevisionNotFoundError,
280 + EntryNotFoundError, LocalEntryNotFoundError) as e:
281 + logger.debug("File or repository not found in hf_hub_download",
282 + e)
283 + return None
284 + file_path = Path(hf_hub_file)
285 +
286 + with open(file_path) as file:
287 + return json.load(file)
288 + return None
289 +
290 +
291 +def get_pooling_config(model: str,
292 + revision: Optional[str] = 'main',
293 + token: Optional[str] = None):
294 + """
295 + This function gets the pooling and normalize
296 + config from the model - only applies to
297 + sentence-transformers models.
298 +
299 + Args:
300 + model (str): The name of the Hugging Face model.
301 + revision (str, optional): The specific version
302 + of the model to use. Defaults to 'main'.
303 +
304 + Returns:
305 + dict: A dictionary containing the pooling
306 + type and whether normalization is used.
307 + """
308 +
309 + modules_file_name = "modules.json"
310 + modules_dict = get_hf_file_to_dict(modules_file_name, model, revision,
311 + token)
312 +
313 + if modules_dict is None:
314 + return None
315 +
316 + pooling = next((item for item in modules_dict
317 + if item["type"] == "sentence_transformers.models.Pooling"),
318 + None)
319 + normalize = bool(
320 + next((item for item in modules_dict
321 + if item["type"] == "sentence_transformers.models.Normalize"),
322 + False))
323 +
324 + if pooling:
325 +
326 + pooling_file_name = "{}/config.json".format(pooling["path"])
327 + pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision,
328 + token)
329 + pooling_type_name = next(
330 + (item for item, val in pooling_dict.items() if val is True), None)
331 +
332 + if pooling_type_name is not None:
333 + pooling_type_name = get_pooling_config_name(pooling_type_name)
334 +
335 + return {"pooling_type": pooling_type_name, "normalize": normalize}
336 +
337 + return None
338 +
339 +
340 +def get_pooling_config_name(pooling_name: str) -> Union[str, None]:
341 + if "pooling_mode_" in pooling_name:
342 + pooling_name = pooling_name.replace("pooling_mode_", "")
343 +
344 + if "_" in pooling_name:
345 + pooling_name = pooling_name.split("_")[0]
346 +
347 + if "lasttoken" in pooling_name:
348 + pooling_name = "last"
349 +
350 + supported_pooling_types = ['LAST', 'ALL', 'CLS', 'STEP', 'MEAN']
351 + pooling_type_name = pooling_name.upper()
352 +
353 + try:
354 + if pooling_type_name in supported_pooling_types:
355 + return pooling_type_name
356 + except NotImplementedError as e:
357 + logger.debug("Pooling type not supported", e)
358 + return None
359 + return None
360 +
361 +
362 +def get_sentence_transformer_tokenizer_config(model: str,
363 + revision: Optional[str] = 'main',
364 + token: Optional[str] = None):
365 + """
366 + Returns the tokenization configuration dictionary for a
367 + given Sentence Transformer BERT model.
368 +
369 + Parameters:
370 + - model (str): The name of the Sentence Transformer
371 + BERT model.
372 + - revision (str, optional): The revision of the m
373 + odel to use. Defaults to 'main'.
374 + - token (str): A Hugging Face access token.
375 +
376 + Returns:
377 + - dict: A dictionary containing the configuration parameters
378 + for the Sentence Transformer BERT model.
379 + """
380 + for config_name in [
381 + "sentence_bert_config.json",
382 + "sentence_roberta_config.json",
383 + "sentence_distilbert_config.json",
384 + "sentence_camembert_config.json",
385 + "sentence_albert_config.json",
386 + "sentence_xlm-roberta_config.json",
387 + "sentence_xlnet_config.json",
388 + ]:
389 + encoder_dict = get_hf_file_to_dict(config_name, model, revision, token)
390 + if encoder_dict:
391 + break
392 +
393 + if not encoder_dict:
394 + return None
395 +
396 + if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
397 + return encoder_dict
398 + return None
399 +
400 +
401 def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
402 """Try to register HF model configuration class to serialize by value
403
@@ -305,20 +460,15 @@ def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
460 exc_info=e)
461
462
308 -def load_params_config(model, revision) -> PretrainedConfig:
463 +def load_params_config(model: Union[str, Path],
464 + revision: Optional[str],
465 + token: Optional[str] = None) -> PretrainedConfig:
466 # This function loads a params.json config which
467 # should be used when loading models in mistral format
468
469 config_file_name = "params.json"
470
314 - config_path = Path(model) / config_file_name
315 -
316 - if not config_path.is_file():
317 - config_path = Path(
318 - hf_hub_download(model, config_file_name, revision=revision))
319 -
320 - with open(config_path) as file:
321 - config_dict = json.load(file)
471 + config_dict = get_hf_file_to_dict(config_file_name, model, revision, token)
472
473 config_mapping = {
474 "dim": "hidden_size",
vllm/transformers_utils/tokenizer_group/__init__.py
+5
@@ -25,6 +25,11 @@ def init_tokenizer_from_configs(model_config: ModelConfig,
25 trust_remote_code=model_config.trust_remote_code,
26 revision=model_config.tokenizer_revision)
27
28 + if (model_config.encoder_config is not None
29 + and "do_lower_case" in model_config.encoder_config):
30 + init_kwargs["do_lower_case"] = model_config.encoder_config[
31 + "do_lower_case"]
32 +
33 return get_tokenizer_group(parallel_config.tokenizer_pool_config,
34 **init_kwargs)
35